diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..ddca09f7f7293a33416d5a5803f02a51aeb46fc2 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test --feature-set context --target utility_margin --k 8 --bootstrap-samples 1000 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee4488d96ff8deabaa33439b5ee33638c24675a1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: context +fit_objective: pointwise +k: 8 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: None +success_bonus: 1.0 +target: utility_margin +threshold_scope: global diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d91fbacd145085f2d9a143f07cf1791ff492f9fc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "130d86b8c5ba5dc901ee48835454693bf42c361803c01f6b2d80590cb2493491", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "88d4bfe8c95f8d1579cbe59096cf665dc5f2dfdd99481a386a25e1f3cfa2b09d", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..58e60527653c049a1edc75cf9002db1286feb6ae --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..2560fcb011a49f27300a482cdabcce90b4890e6e --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics.json @@ -0,0 +1,8987 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.3506944444444444, + "fallback_rate": 0.6493055555555556, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3055555555555556, + "selected_success_gain_over_base": 0.006944444444444444, + "selected_utility": 0.7529713824042119, + "selector_regret": 0.07015972012757427, + "success_selector_gap": 0.03819444444444445, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_summary": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.3506944444444444, + "fallback_rate": 0.6493055555555556, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "pairwise_causal_calibration_accuracy": 0.5068881685575365, + "pairwise_causal_calibration_confidence": 0.5128815859618687, + "pairwise_causal_calibration_ece": 0.005993417404332235, + "pairwise_causal_calibration_pairs": 7404.0, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3055555555555556, + "selected_success_gain_over_base": 0.006944444444444444, + "selected_utility": 0.7529713824042119, + "selector_regret": 0.07015972012757427, + "success_selector_gap": 0.03819444444444445, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.3333333333333333, + "base_utility": 0.8338337515791258, + "coverage": 0.4236111111111111, + "fallback_rate": 0.5763888888888888, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.3611111111111111, + "pairwise_causal_calibration_accuracy": 0.5480611687602404, + "pairwise_causal_calibration_confidence": 0.513613968802469, + "pairwise_causal_calibration_ece": 0.034447199957771346, + "pairwise_causal_calibration_pairs": 3662.0, + "proposal_oracle_success": 0.2222222222222222, + "proposal_oracle_utility": 0.6232009150553495, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.830257743739316, + "selector_regret": 0.11280556699623251, + "success_selector_gap": 0.0625, + "success_support_gap": 0.5138888888888888, + "support_gap": 0.9091424554741631 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.182546837171281, + 1.8836221616495297, + -1.0601087908747155e-18, + 3.5, + 0.125, + 1.2092013888888888, + 0.7583173435164257, + 1.9059527259604592, + 8.0, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.36258169934639883, + -0.4588235294117674, + 0.1346405228758154, + 0.3127450980392166, + -0.5231481481481283, + -0.5494008714596817, + 0.5098039215686088, + -0.08622004357298363, + 1.0, + 0.2707248263888889, + 0.2306857638888889 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "target_task_hash_00", + "target_task_hash_01", + "target_task_hash_02", + "target_task_hash_03", + "target_task_hash_04", + "target_task_hash_05", + "target_task_hash_06", + "target_task_hash_07", + "source_task_hash_00", + "source_task_hash_01", + "source_task_hash_02", + "source_task_hash_03", + "source_task_hash_04", + "source_task_hash_05", + "source_task_hash_06", + "source_task_hash_07", + "instruction_hash_00", + "instruction_hash_01", + "instruction_hash_02", + "instruction_hash_03", + "instruction_hash_04", + "instruction_hash_05", + "instruction_hash_06", + "instruction_hash_07", + "source_target_same_task", + "instruction_chars_per_128", + "instruction_words_per_32" + ], + "feature_set": "context", + "feature_std": [ + 1.0, + 2.4903288136152946, + 2.1478743840800227, + 0.999998700632758, + 2.29128884747792, + 0.3307199138830738, + 1.490882562848811, + 0.3689689239790769, + 0.7063143432867478, + 1e-06, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37487873363847424, + 0.3443939715604908, + 0.32513795823134695, + 0.3102098787644046, + 0.4154730332740044, + 0.3938110555196154, + 0.4628208676716171, + 0.7177468630079228, + 1e-06, + 0.03103759015348271, + 0.02828865844409628 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 2304, + "num_fit_rows": 2304, + "num_pairwise_rows": 0, + "num_pointwise_rows": 2304, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 8, + "num_calibration_candidates": 2304, + "num_calibration_rows": 288, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5068881685575365, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.005993417404332235, + "accuracy": 0.5068881685575365, + "confidence": 0.5128815859618687, + "count": 7404, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.005993417404332235, + "mean_confidence": 0.5128815859618687, + "n_bins": 10, + "num_pairs": 7404, + "num_rows": 288 + }, + "eval": { + "accuracy": 0.5480611687602404, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.034447199957771346, + "accuracy": 0.5480611687602404, + "confidence": 0.513613968802469, + "count": 3662, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.034447199957771346, + "mean_confidence": 0.513613968802469, + "n_bins": 10, + "num_pairs": 3662, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5077128468462638, + "pairwise_causal_calibration_ece": 0.20657286743945047, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15769274322565577, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "seed": "2012211287/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5194284844280975, + "pairwise_causal_calibration_ece": 0.016285801286188173, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.031234880094876627, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "seed": "307068879/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19984664022922516, + "selected_utility_gain_over_base": -0.009305715560913086, + "selector_regret": 0.0029561370611190796, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5092955414174704, + "pairwise_causal_calibration_ece": 0.009295541417470354, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05895485055674793, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "seed": "1073161778/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7119815349578857, + "selected_utility_gain_over_base": -0.013536453247070312, + "selector_regret": 0.006028711795806885, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5078873058739054, + "pairwise_causal_calibration_ece": 0.027826979840380295, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04985292651651213, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "seed": "608016290/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5531623363494873, + "selected_utility_gain_over_base": 0.027802228927612305, + "selector_regret": 0.05929923057556152, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5195701592812082, + "pairwise_causal_calibration_ece": 0.1590012692902204, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.02945650465411806, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "seed": "441046872/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.203963503241539, + "selected_utility_gain_over_base": -0.005804136395454407, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5319219008021803, + "pairwise_causal_calibration_ece": 0.07522095634067683, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3891433321605168, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "seed": "1522272686/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "task_id": "PickCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.3331167533441346, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5058430224913567, + "pairwise_causal_calibration_ece": 0.36298587963421386, + "pairwise_causal_calibration_pairs": 7.0, + "predicted_margin": -0.15034784448971583, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5091120840652484, + "pairwise_causal_calibration_ece": 0.009112084065248438, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.12922901502057813, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "seed": "1461069713/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5080295100043231, + "pairwise_causal_calibration_ece": 0.38482763285281985, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.21562654428916522, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.044337254017591476, + "seed": "626486813/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01164356991648674, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2090834565460682, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5052995351748154, + "pairwise_causal_calibration_ece": 0.041013820889101105, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.511525060934223, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14525407552719116, + "seed": "2054899126/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8547459244728088, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5213462849964556, + "pairwise_causal_calibration_ece": 0.19293942928925867, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1294553020456159, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "seed": "672328262/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.045009810477495193, + "selected_utility_gain_over_base": -0.060924429446458817, + "selector_regret": 0.014783523976802826, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5157932993620346, + "pairwise_causal_calibration_ece": 0.015793299362034552, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.32446523226392154, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2748139500617981, + "seed": "767747424/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07782270014286041, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5087559775718801, + "pairwise_causal_calibration_ece": 0.02695830814240563, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.060780128983826176, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14440619945526123, + "seed": "278008524/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09351998567581177, + "selected_utility_gain_over_base": -0.15649443864822388, + "selector_regret": 0.05088621377944946, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0834110826253891, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5131863087960253, + "pairwise_causal_calibration_ece": 0.022527976918260362, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.056123788714222106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "seed": "1845489013/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04260633513331413, + "selected_utility_gain_over_base": -0.06995167210698128, + "selector_regret": 0.00027813389897346497, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.03571428571428571, + "pairwise_causal_calibration_confidence": 0.5054723459936693, + "pairwise_causal_calibration_ece": 0.4697580602793836, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.046439836721598904, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07115759700536728, + "seed": "626486813/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03407871723175049, + "selected_utility_gain_over_base": -0.01670938730239868, + "selector_regret": 0.03707887977361679, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.006586454808712006, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5195462389333972, + "pairwise_causal_calibration_ece": 0.19473947535231706, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3157182439831808, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.011163711547851562, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2515503764152527, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5181208856251349, + "pairwise_causal_calibration_ece": 0.23187911437486508, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03414964989475239, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "seed": "585904130/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14155812561511993, + "selected_utility_gain_over_base": 0.07052158564329147, + "selector_regret": 0.022925466299057007, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.5189149557338043, + "pairwise_causal_calibration_ece": 0.4096564728376243, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.13505606653156377, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "seed": "141984615/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5165016398183268, + "pairwise_causal_calibration_ece": 0.054926931610244645, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.056828534011851145, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25842753052711487, + "seed": "750448296/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.23790733516216278, + "selected_utility_gain_over_base": 0.16980473697185516, + "selector_regret": 0.020520195364952087, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7415724694728851, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5055966367137085, + "pairwise_causal_calibration_ece": 0.17297479185772013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11026859162288982, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "seed": "636680137/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5082708048263679, + "pairwise_causal_calibration_ece": 0.2939850905406536, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11390252283770846, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.029627859592437744, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.16747966408729553, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.505879116784551, + "pairwise_causal_calibration_ece": 0.1981868090922433, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.28547700247176366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5257244389989378, + "pairwise_causal_calibration_ece": 0.33141841814391926, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18263322741939308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "seed": "2038594191/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15247176587581635, + "selected_utility_gain_over_base": 0.0574311688542366, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5175610583130501, + "pairwise_causal_calibration_ece": 0.1252960845440928, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.13136965043797189, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13244229555130005, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09154719114303589, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.12386479553124777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5050963819862415, + "pairwise_causal_calibration_ece": 0.14795352484338437, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.41293810625522775, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "seed": "747260437/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.09020996114341928, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5262312992720155, + "pairwise_causal_calibration_ece": 0.3666258435851274, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.048325782527749814, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21791090071201324, + "selected_utility_gain_over_base": 0.04541558027267456, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5154582541421188, + "pairwise_causal_calibration_ece": 0.19882746014359554, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09172847339145515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "seed": "1822210471/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03715964034199715, + "selected_utility_gain_over_base": -0.05726582184433937, + "selector_regret": 0.0008668527007102966, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5189249165244855, + "pairwise_causal_calibration_ece": 0.23321063081019977, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3853237478359026, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23744209110736847, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7625579088926315, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5057012649277024, + "pairwise_causal_calibration_ece": 0.2800130207865833, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0254871904912983, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "seed": "440577711/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5138598359008081, + "pairwise_causal_calibration_ece": 0.08528840732937953, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.2895725332761157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "seed": "1324642411/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5057916478058077, + "pairwise_causal_calibration_ece": 0.31563692362276374, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0792771393876592, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5116618738353578, + "pairwise_causal_calibration_ece": 0.024052411878927926, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.042599098629623, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "seed": "1261930456/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.5063024493634376, + "pairwise_causal_calibration_ece": 0.1336975506365624, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -1.155476122058963, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5083154135037583, + "pairwise_causal_calibration_ece": 0.09882744363909879, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.1191024751801397, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "seed": "1480856810/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5199008764839764, + "pairwise_causal_calibration_ece": 0.19438483780173788, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03336441156547287, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "seed": "2061635888/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07517768442630768, + "selected_utility_gain_over_base": -0.011029243469238281, + "selector_regret": 0.0012469589710235596, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5166331169260667, + "pairwise_causal_calibration_ece": 0.23336688307393327, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0781444029185483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "seed": "655884605/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01332351565361023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5211029637363088, + "pairwise_causal_calibration_ece": 0.08603989340654827, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04825948731449364, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0680118203163147, + "selected_utility_gain_over_base": -0.043087467551231384, + "selector_regret": 0.0060281902551651, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5123855862850035, + "pairwise_causal_calibration_ece": 0.19095701485643207, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07399457979836796, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "seed": "1578990937/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21093030273914337, + "selected_utility_gain_over_base": -0.0033356696367263794, + "selector_regret": 0.003326788544654846, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.5128660165598996, + "pairwise_causal_calibration_ece": 0.1928660165598996, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.14698878641541877, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5086109136222222, + "pairwise_causal_calibration_ece": 0.06281765780634918, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.133838155897241, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "seed": "211245842/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3979237377643585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5131067749938627, + "pairwise_causal_calibration_ece": 0.2631067749938627, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06960468013481613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "seed": "1151410346/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.23053553700447083, + "selected_utility_gain_over_base": 0.0020620226860046387, + "selector_regret": 0.337958961725235, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5224112709742389, + "pairwise_causal_calibration_ece": 0.022411270974238895, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09201776605615172, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "seed": "1324133122/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5100395188896416, + "pairwise_causal_calibration_ece": 0.1328176239675013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1980299443706942, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "seed": "701667486/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5062670117038048, + "pairwise_causal_calibration_ece": 0.18483844027523338, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.16113806347993082, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5109113095795552, + "pairwise_causal_calibration_ece": 0.16766011899187339, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11814581215768769, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "seed": "1999022574/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7777777777777778, + "pairwise_causal_calibration_confidence": 0.5098868256616201, + "pairwise_causal_calibration_ece": 0.26789095211615765, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": -0.12821380008018857, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2717533111572266, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5225141037494058, + "pairwise_causal_calibration_ece": 0.12034303910773714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.037469066617763, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "seed": "307068879/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20222051441669464, + "selected_utility_gain_over_base": -0.0069318413734436035, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5121418326693615, + "pairwise_causal_calibration_ece": 0.02357245304492417, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06517628519498339, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "seed": "1073161778/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.713421642780304, + "selected_utility_gain_over_base": -0.0120963454246521, + "selector_regret": 7.271766662597656e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5071201830818504, + "pairwise_causal_calibration_ece": 0.1499773259389932, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05236386089192557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "seed": "608016290/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5450301170349121, + "selected_utility_gain_over_base": 0.01967000961303711, + "selector_regret": 0.06507062911987305, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5215255698164627, + "pairwise_causal_calibration_ece": 0.2284744301835373, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05890578182706513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "seed": "441046872/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20609354972839355, + "selected_utility_gain_over_base": -0.0036740899085998535, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5176703997119507, + "pairwise_causal_calibration_ece": 0.2676703997119507, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3649738510562886, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "seed": "1522272686/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "task_id": "PickCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.38909781967835777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5121260545775028, + "pairwise_causal_calibration_ece": 0.08355462600607427, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.18068902910685475, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5081015084720596, + "pairwise_causal_calibration_ece": 0.3133270629565118, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1801597858223893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "seed": "1461069713/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.508544019970645, + "pairwise_causal_calibration_ece": 0.24145598002935498, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11017436819471127, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06662949174642563, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03393580764532089, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18679121881723404, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5105702026856556, + "pairwise_causal_calibration_ece": 0.1891416312570841, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3885007143857401, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1511060744524002, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8488939255475998, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5156576148233183, + "pairwise_causal_calibration_ece": 0.08708618625188974, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09180646718725743, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "seed": "672328262/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05302006006240845, + "selected_utility_gain_over_base": -0.05291417986154556, + "selector_regret": 0.00036839768290519714, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5067825918204408, + "pairwise_causal_calibration_ece": 0.028931693893844868, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.32395301977780033, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2738197147846222, + "seed": "767747424/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07682846486568451, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5071032645290914, + "pairwise_causal_calibration_ece": 0.17146816404233722, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.01203132006527452, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24920666217803955, + "seed": "278008524/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22021572291851044, + "selected_utility_gain_over_base": -0.029798701405525208, + "selector_regret": 0.028990939259529114, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5137646328948755, + "pairwise_causal_calibration_ece": 0.1290925099622674, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10685267161815218, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "seed": "1845489013/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06991276890039444, + "selected_utility_gain_over_base": -0.04264523833990097, + "selector_regret": 0.025380156934261322, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5045593185796378, + "pairwise_causal_calibration_ece": 0.004559318579637761, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03869238687687371, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05724402144551277, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05345620587468147, + "selected_utility_gain_over_base": 0.002668101340532303, + "selector_regret": 0.003787815570831299, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.020500030368566513, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5101548807306857, + "pairwise_causal_calibration_ece": 0.061273690697885685, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2329955845615604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.013422667980194092, + "seed": "767747424/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24929141998291016, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.512435265258071, + "pairwise_causal_calibration_ece": 0.048149550972356736, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.027362083160849926, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "seed": "585904130/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12263611704111099, + "selected_utility_gain_over_base": 0.05159957706928253, + "selector_regret": 0.034353144466876984, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5139341766826016, + "pairwise_causal_calibration_ece": 0.013934176682601551, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05316347180765005, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04150080680847168, + "selected_utility_gain_over_base": -0.12633192539215088, + "selector_regret": 0.002477247267961502, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5225911107343371, + "pairwise_causal_calibration_ece": 0.1654482535914799, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.007860316442232548, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2347063571214676, + "seed": "750448296/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17607416212558746, + "selected_utility_gain_over_base": 0.10797156393527985, + "selector_regret": 0.05863219499588013, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7652936428785324, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5078518602413418, + "pairwise_causal_calibration_ece": 0.11499471738419892, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14666728076214766, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "seed": "636680137/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5220065693852253, + "pairwise_causal_calibration_ece": 0.20057799795665382, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07324658798131045, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04315249249339104, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030954698100686073, + "selected_utility_gain_over_base": -0.070889076218009, + "selector_regret": 0.012197794392704964, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15395503118634224, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5120529709093987, + "pairwise_causal_calibration_ece": 0.02366131480488698, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3533321141266833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "seed": "1375492786/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.514770372402238, + "pairwise_causal_calibration_ece": 0.264770372402238, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1571311927054344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "seed": "2038594191/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14092111587524414, + "selected_utility_gain_over_base": 0.0458805188536644, + "selector_regret": 0.049359604716300964, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5124291169677748, + "pairwise_causal_calibration_ece": 0.11957197411063197, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.048698938405898395, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23850953578948975, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1983291506767273, + "selected_utility_gain_over_base": -0.015078231692314148, + "selector_regret": 0.04018038511276245, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.06200148769583112, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.5172521843522638, + "pairwise_causal_calibration_ece": 0.41131924421916477, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.43548232619379196, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.02641892029001297, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5173089481291107, + "pairwise_causal_calibration_ece": 0.26840533758517504, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06964181922005985, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.3088321089744568, + "selected_utility_gain_over_base": 0.1363367885351181, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5122846916174123, + "pairwise_causal_calibration_ece": 0.059143879811159095, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11469903634944659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06487039476633072, + "selected_utility_gain_over_base": -0.0295550674200058, + "selector_regret": 0.010523699223995209, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5173797001049291, + "pairwise_causal_calibration_ece": 0.23166541439064336, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.29830924231344125, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2027529925107956, + "seed": "750448296/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7972470074892044, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.505418210717456, + "pairwise_causal_calibration_ece": 0.19772590302514825, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -1.067727630841523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4074074074074074, + "pairwise_causal_calibration_confidence": 0.516058262207234, + "pairwise_causal_calibration_ece": 0.10865085479982667, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": -1.1498268645090013, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5062566081726894, + "pairwise_causal_calibration_ece": 0.10088624897016774, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0821139696201234, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "seed": "1790188787/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5139278716117496, + "pairwise_causal_calibration_ece": 0.15678501446889243, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0195808227095515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "seed": "1261930456/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5086436934457946, + "pairwise_causal_calibration_ece": 0.21862903382693266, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.106462385735648, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5110030837863524, + "pairwise_causal_calibration_ece": 0.15386022664349525, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.1493088895471595, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5189283527559876, + "pairwise_causal_calibration_ece": 0.15964307581544102, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04191814555358769, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "seed": "2061635888/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08908586204051971, + "selected_utility_gain_over_base": 0.002878934144973755, + "selector_regret": 0.0013788342475891113, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5226674723877733, + "pairwise_causal_calibration_ece": 0.09409604381634479, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.01818175818777909, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "seed": "655884605/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12485501170158386, + "selected_utility_gain_over_base": -0.01922038197517395, + "selector_regret": 0.025532275438308716, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5243376283502783, + "pairwise_causal_calibration_ece": 0.13148048549313546, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05766058105545785, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "seed": "1113371261/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08607026934623718, + "selected_utility_gain_over_base": -0.0250290185213089, + "selector_regret": 0.010152161121368408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5156994405981151, + "pairwise_causal_calibration_ece": 0.23430055940188488, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06039227699080513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "seed": "1578990937/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21256697177886963, + "selected_utility_gain_over_base": -0.001699000597000122, + "selector_regret": 0.00506339967250824, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.52, + "pairwise_causal_calibration_confidence": 0.511941221071365, + "pairwise_causal_calibration_ece": 0.008058778928635046, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.09190610579492059, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.507099429830539, + "pairwise_causal_calibration_ece": 0.07852800125911047, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14019095394361425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "seed": "211245842/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.017371907830238342, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5141935806753487, + "pairwise_causal_calibration_ece": 0.3356221521039201, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06131251512798924, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "seed": "1151410346/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2249394953250885, + "selected_utility_gain_over_base": -0.0035340189933776855, + "selector_regret": 0.01898777484893799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5213408711746942, + "pairwise_causal_calibration_ece": 0.08580198596816291, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07239433850689977, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "seed": "1324133122/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06854432821273804, + "selected_utility_gain_over_base": -0.015682727098464966, + "selector_regret": 0.015829041600227356, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5074799238651517, + "pairwise_causal_calibration_ece": 0.0074799238651517275, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1667987048169856, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "seed": "701667486/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5113786791180399, + "pairwise_causal_calibration_ece": 0.2386213208819601, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15811820951338246, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.517373533290843, + "pairwise_causal_calibration_ece": 0.017373533290843013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08196834306253598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5100908012774384, + "pairwise_causal_calibration_ece": 0.38276634157970446, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09296986205333337, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "seed": "2012211287/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.518763232568162, + "pairwise_causal_calibration_ece": 0.19552248171755227, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.030969288861405195, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "seed": "307068879/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2035302072763443, + "selected_utility_gain_over_base": -0.005622148513793945, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5109233051180221, + "pairwise_causal_calibration_ece": 0.024790980596263612, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03661228926063653, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7243968844413757, + "selected_utility_gain_over_base": -0.0011211037635803223, + "selector_regret": 1.2756031155586243, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5096252566953113, + "pairwise_causal_calibration_ece": 0.16894617187611727, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.012997582312052308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "seed": "608016290/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.4804247319698334, + "selected_utility_gain_over_base": -0.044935375452041626, + "selector_regret": 0.09008613228797913, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5163446864153005, + "pairwise_causal_calibration_ece": 0.09079817072755658, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.017015444441914704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "seed": "441046872/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20165641605854034, + "selected_utility_gain_over_base": -0.008111223578453064, + "selector_regret": 0.002608165144920349, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "task_id": "LiftPegUpright-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5183770201500114, + "pairwise_causal_calibration_ece": 0.05409130586429711, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.35603713675787785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1522272686/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PickCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.46100102368298207, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5077277139880857, + "pairwise_causal_calibration_ece": 0.2422722860119143, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.24580711842037567, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5120292310110632, + "pairwise_causal_calibration_ece": 0.023685054703222463, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.02022527141535374, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "seed": "1461069713/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09350365400314331, + "selected_utility_gain_over_base": -0.13502393662929535, + "selector_regret": 0.02192588895559311, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.513637691536636, + "pairwise_causal_calibration_ece": 0.09350516560622113, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.13237150311100343, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10755517333745956, + "seed": "626486813/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07486148923635483, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1458655372262001, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.513192964083336, + "pairwise_causal_calibration_ece": 0.12966417877380687, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.5062143003990996, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13674892485141754, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8632510751485825, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5131890723327346, + "pairwise_causal_calibration_ece": 0.022525213381551112, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11194999822009422, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "seed": "672328262/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0664752721786499, + "selected_utility_gain_over_base": -0.03945896774530411, + "selector_regret": 0.005690954625606537, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5098224520361478, + "pairwise_causal_calibration_ece": 0.1883938806075764, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.27319331674968994, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25252875685691833, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05553750693798065, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5171398278381145, + "pairwise_causal_calibration_ece": 0.3028541135524002, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.027899109682451684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.31406447291374207, + "seed": "278008524/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.231573224067688, + "selected_utility_gain_over_base": -0.018441200256347656, + "selector_regret": 0.08249124884605408, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5162114675678457, + "pairwise_causal_calibration_ece": 0.30521710386072565, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11053198394490094, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "seed": "1845489013/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0861951932311058, + "selected_utility_gain_over_base": -0.026362814009189606, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5112026895776399, + "pairwise_causal_calibration_ece": 0.20308302470807438, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028815968843147743, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05257529020309448, + "seed": "626486813/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05257529020309448, + "selected_utility_gain_over_base": 0.0017871856689453125, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.025168761610984802, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5229148429555205, + "pairwise_causal_calibration_ece": 0.04851372847305091, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.20057429936871213, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.01747530698776245, + "seed": "767747424/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2452387809753418, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5094731430629448, + "pairwise_causal_calibration_ece": 0.2951874287772305, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.026060848101647938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13704223930835724, + "selected_utility_gain_over_base": 0.06600569933652878, + "selector_regret": 0.15684272348880768, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.505048424591939, + "pairwise_causal_calibration_ece": 0.005048424591938994, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.10105758631461294, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "seed": "141984615/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.512135294396181, + "pairwise_causal_calibration_ece": 0.262135294396181, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1024792174890061, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.4033660888671875, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3352634906768799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.5966339111328125, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5081316115880623, + "pairwise_causal_calibration_ece": 0.3132969598405091, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08206888832985737, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "seed": "636680137/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005137942731380463, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5124138521488978, + "pairwise_causal_calibration_ece": 0.36955670929175494, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03287953871375404, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053976815193891525, + "seed": "767747424/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02692335844039917, + "selected_utility_gain_over_base": -0.0749204158782959, + "selector_regret": 0.027053456753492355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14313070848584175, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5103681394924954, + "pairwise_causal_calibration_ece": 0.2753461462217903, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.38601214633080416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "seed": "1375492786/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5169917573441141, + "pairwise_causal_calibration_ece": 0.12413461448697122, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1499261458574031, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "seed": "2038594191/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10970946401357651, + "selected_utility_gain_over_base": 0.014668866991996765, + "selector_regret": 0.1390988454222679, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5079772209141769, + "pairwise_causal_calibration_ece": 0.1508343637713197, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09508391083289638, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17124910652637482, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05274038016796112, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.3009593411123786, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5073556110406273, + "pairwise_causal_calibration_ece": 0.06407296038794408, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.44021912627823445, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "seed": "747260437/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.0479044142283129, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5161816159310763, + "pairwise_causal_calibration_ece": 0.09096124121178084, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0581167593221707, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "seed": "1549324805/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12139803171157837, + "selected_utility_gain_over_base": -0.051097288727760315, + "selector_regret": 0.09263791143894196, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5177920443784734, + "pairwise_causal_calibration_ece": 0.12506509847866953, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11123534194819423, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "seed": "1822210471/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.057336073368787766, + "selected_utility_gain_over_base": -0.03708938881754875, + "selector_regret": 0.0014832988381385803, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5251723292470215, + "pairwise_causal_calibration_ece": 0.16802947210416436, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3926176776006134, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1698276251554489, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.830172374844551, + "task_id": "PullCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.9797113666967447, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7222222222222222, + "pairwise_causal_calibration_confidence": 0.5090482166780169, + "pairwise_causal_calibration_ece": 0.2131740055442053, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": -1.1472572709125655, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8636363636363636, + "pairwise_causal_calibration_confidence": 0.5082783651389248, + "pairwise_causal_calibration_ece": 0.3553579984974389, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.0426295433413653, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5121665051198715, + "pairwise_causal_calibration_ece": 0.1306906377372714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0158058232150635, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "seed": "1261930456/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.5071930884530261, + "pairwise_causal_calibration_ece": 0.08371600245606481, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.0541420784393922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5118724897465404, + "pairwise_causal_calibration_ece": 0.05955608168203097, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0926696735019097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "seed": "1480856810/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "task_id": "PushCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5201168018122745, + "pairwise_causal_calibration_ece": 0.16297394466941734, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.033653298036953966, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "seed": "2061635888/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07194386422634125, + "selected_utility_gain_over_base": -0.014263063669204712, + "selector_regret": 0.00708027184009552, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.525781459845288, + "pairwise_causal_calibration_ece": 0.025781459845288013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.024273861804607055, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "seed": "655884605/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16552868485450745, + "selected_utility_gain_over_base": 0.021453291177749634, + "selector_regret": 0.0068964362144470215, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5258702008499507, + "pairwise_causal_calibration_ece": 0.15270122772147787, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04967133196095555, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "seed": "1113371261/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09928342700004578, + "selected_utility_gain_over_base": -0.011815860867500305, + "selector_regret": 0.0010813623666763306, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5121377255628292, + "pairwise_causal_calibration_ece": 0.012137725562829216, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09822142936424738, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "seed": "1578990937/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018326029181480408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5103616032224382, + "pairwise_causal_calibration_ece": 0.08179017465100963, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1520858341588262, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "seed": "1509980664/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5123986202425596, + "pairwise_causal_calibration_ece": 0.13045852261458335, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0967227357905851, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "seed": "211245842/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.029980570077896118, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5140320226786702, + "pairwise_causal_calibration_ece": 0.19260345125009876, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.054743002594082435, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "seed": "1151410346/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.23354609310626984, + "selected_utility_gain_over_base": 0.00507257878780365, + "selector_regret": 0.32946430146694183, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5254412617086446, + "pairwise_causal_calibration_ece": 0.18884445257706972, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05650027981133776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "seed": "1324133122/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09103855490684509, + "selected_utility_gain_over_base": 0.00681149959564209, + "selector_regret": 0.007112905383110046, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5089535245646244, + "pairwise_causal_calibration_ece": 0.1696179040068042, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.19998664785779915, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5089676132647365, + "pairwise_causal_calibration_ece": 0.2232533275504508, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.18531182086534548, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "seed": "412102563/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5201361575280914, + "pairwise_causal_calibration_ece": 0.1941495567576229, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09926036807855881, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "seed": "1999022574/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "task_id": "StackCube-v1", + "tau": -0.07465520951292609, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 0.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3958333333333333, + "low": 0.2277777777777778, + "mean": 0.3083333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3083333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8053333333333332, + "low": 0.10836363636363637, + "mean": 0.45684848484848484, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.45684848484848484, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.4097222222222222, + "low": 0.2569444444444444, + "mean": 0.3333333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9566364926286041, + "low": 0.6609309806488455, + "mean": 0.8028034761858482, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.8028034761858482, + "per_seed": { + "1073161778/train0": 0.725517988204956, + "1073161778/train1": 0.725517988204956, + "1073161778/train2": 0.725517988204956, + "1113371261/train0": 0.11109928786754608, + "1113371261/train1": 0.11109928786754608, + "1113371261/train2": 0.11109928786754608, + "1151410346/train0": 0.2284735143184662, + "1151410346/train1": 0.2284735143184662, + "1151410346/train2": 0.2284735143184662, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.084227055311203, + "1324133122/train2": 0.084227055311203, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.16783273220062256, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.22852759063243866, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.17249532043933868, + "1549324805/train1": 0.17249532043933868, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21426597237586975, + "1578990937/train1": 0.21426597237586975, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.09442546218633652, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.09442546218633652, + "1845489013/train0": 0.11255800724029541, + "1845489013/train1": 0.11255800724029541, + "1845489013/train2": 0.11255800724029541, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.09504059702157974, + "2038594191/train1": 0.09504059702157974, + "2038594191/train2": 0.09504059702157974, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.08620692789554596, + "2061635888/train1": 0.08620692789554596, + "2061635888/train2": 0.08620692789554596, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.1250072121620178, + "278008524/train1": 1.1250072121620178, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.20915235579013824, + "307068879/train1": 0.20915235579013824, + "307068879/train2": 0.20915235579013824, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.2097676396369934, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2097676396369934, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.07103653997182846, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.07103653997182846, + "608016290/train0": 0.525360107421875, + "608016290/train1": 0.525360107421875, + "608016290/train2": 0.525360107421875, + "626486813/train0": 0.04174089431762695, + "626486813/train1": 0.04174089431762695, + "626486813/train2": 0.04174089431762695, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.1440753936767578, + "655884605/train2": 0.1440753936767578, + "672328262/train0": 0.10593423992395401, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.10593423992395401, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7605033268531164, + "750448296/train1": 0.7605033268531164, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0747087560594082, + "767747424/train1": 1.0747087560594082, + "767747424/train2": 1.0747087560594082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6347634333101186, + "low": 0.5526382644826715, + "mean": 1.0913353469100866, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.0913353469100866, + "per_task": { + "LiftPegUpright-v1": 0.4796089559793472, + "PickCube-v1": 2.0, + "PullCube-v1": 0.8032506120204925, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.5003613992170854 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9699174793851044, + "low": 0.6952519603073597, + "mean": 0.8338337515791258, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5229166666666667, + "low": 0.3541666666666667, + "mean": 0.44166666666666665, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.44166666666666665, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5927272727272728, + "low": 0.09066666666666666, + "mean": 0.3415757575757576, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3415757575757576, + "per_task": { + "LiftPegUpright-v1": 0.8, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4533333333333333, + "PushCube-v1": 0.0, + "StackCube-v1": 0.45454545454545453 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.5069444444444444, + "low": 0.3472222222222222, + "mean": 0.4236111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5229166666666667, + "low": 0.3541666666666667, + "mean": 0.44166666666666665, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.44166666666666665, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5927272727272728, + "low": 0.09066666666666666, + "mean": 0.3415757575757576, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3415757575757576, + "per_task": { + "LiftPegUpright-v1": 0.8, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4533333333333333, + "PushCube-v1": 0.0, + "StackCube-v1": 0.45454545454545453 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.5069444444444444, + "low": 0.3472222222222222, + "mean": 0.4236111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.47708333333333325, + "mean": 0.5583333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5583333333333333, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 0.5, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 1.0, + "767747424/train0": 1.0, + "767747424/train1": 0.75, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9093333333333333, + "low": 0.40727272727272723, + "mean": 0.6584242424242424, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6584242424242424, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 1.0, + "PullCube-v1": 0.5466666666666666, + "PushCube-v1": 1.0, + "StackCube-v1": 0.5454545454545454 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.6527777777777778, + "low": 0.4930555555555556, + "mean": 0.5763888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.45138888888888895, + "low": 0.2833333333333333, + "mean": 0.3631944444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3631944444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 1.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4609696969696969, + "low": 0.1, + "mean": 0.2852121212121212, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2852121212121212, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.6060606060606061 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.4444444444444444, + "low": 0.2847222222222222, + "mean": 0.3611111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5922108287625532, + "low": 0.5306914062948546, + "mean": 0.5607560814026332, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.5607560814026332, + "per_seed": { + "1073161778/train0": 0.5, + "1073161778/train1": 0.5357142857142857, + "1073161778/train2": 0.5357142857142857, + "1113371261/train0": 0.6071428571428571, + "1113371261/train1": 0.39285714285714285, + "1113371261/train2": 0.6785714285714286, + "1151410346/train0": 0.25, + "1151410346/train1": 0.17857142857142858, + "1151410346/train2": 0.32142857142857145, + "1261930456/train0": 0.5357142857142857, + "1261930456/train1": 0.35714285714285715, + "1261930456/train2": 0.6428571428571429, + "1324133122/train0": 0.5, + "1324133122/train1": 0.6071428571428571, + "1324133122/train2": 0.7142857142857143, + "1324642411/train0": 0.42857142857142855, + "1324642411/train1": 0.4074074074074074, + "1324642411/train2": 0.7222222222222222, + "1375492786/train0": 0.3076923076923077, + "1375492786/train1": 0.5357142857142857, + "1375492786/train2": 0.7857142857142857, + "141984615/train0": 0.9285714285714286, + "141984615/train1": 0.5, + "141984615/train2": 0.5, + "1461069713/train0": 0.5, + "1461069713/train1": 0.8214285714285714, + "1461069713/train2": 0.5357142857142857, + "1480856810/train0": 0.6071428571428571, + "1480856810/train1": 0.35714285714285715, + "1480856810/train2": 0.5714285714285714, + "1509980664/train0": 0.32, + "1509980664/train1": 0.52, + "1509980664/train2": 0.42857142857142855, + "1522272686/train0": 0.6071428571428571, + "1522272686/train1": 0.25, + "1522272686/train2": 0.4642857142857143, + "1549324805/train0": 0.8928571428571429, + "1549324805/train1": 0.7857142857142857, + "1549324805/train2": 0.6071428571428571, + "1578990937/train0": 0.32142857142857145, + "1578990937/train1": 0.75, + "1578990937/train2": 0.5, + "1790188787/train0": 0.8214285714285714, + "1790188787/train1": 0.6071428571428571, + "1790188787/train2": 0.8636363636363636, + "1822210471/train0": 0.7142857142857143, + "1822210471/train1": 0.5714285714285714, + "1822210471/train2": 0.6428571428571429, + "1845489013/train0": 0.5357142857142857, + "1845489013/train1": 0.6428571428571429, + "1845489013/train2": 0.8214285714285714, + "1999022574/train0": 0.6785714285714286, + "1999022574/train1": 0.5, + "1999022574/train2": 0.7142857142857143, + "2012211287/train0": 0.7142857142857143, + "2012211287/train1": 0.7777777777777778, + "2012211287/train2": 0.8928571428571429, + "2038594191/train0": 0.8571428571428571, + "2038594191/train1": 0.25, + "2038594191/train2": 0.39285714285714285, + "2054899126/train0": 0.4642857142857143, + "2054899126/train1": 0.32142857142857145, + "2054899126/train2": 0.6428571428571429, + "2061635888/train0": 0.7142857142857143, + "2061635888/train1": 0.6785714285714286, + "2061635888/train2": 0.35714285714285715, + "211245842/train0": 0.5714285714285714, + "211245842/train1": 0.42857142857142855, + "211245842/train2": 0.6428571428571429, + "278008524/train0": 0.5357142857142857, + "278008524/train1": 0.6785714285714286, + "278008524/train2": 0.21428571428571427, + "307068879/train0": 0.5357142857142857, + "307068879/train1": 0.6428571428571429, + "307068879/train2": 0.7142857142857143, + "412102563/train0": 0.32142857142857145, + "412102563/train1": 0.75, + "412102563/train2": 0.2857142857142857, + "440577711/train0": 0.7857142857142857, + "440577711/train1": 0.3076923076923077, + "441046872/train0": 0.6785714285714286, + "441046872/train1": 0.75, + "441046872/train2": 0.6071428571428571, + "509430392/train0": 0.64, + "509430392/train1": 0.7272727272727273, + "509430392/train2": 0.5909090909090909, + "585904130/train0": 0.75, + "585904130/train1": 0.4642857142857143, + "585904130/train2": 0.21428571428571427, + "608016290/train0": 0.5357142857142857, + "608016290/train1": 0.35714285714285715, + "608016290/train2": 0.6785714285714286, + "626486813/train0": 0.4642857142857143, + "626486813/train1": 0.625, + "626486813/train2": 0.6607142857142857, + "636680137/train0": 0.6785714285714286, + "636680137/train1": 0.39285714285714285, + "636680137/train2": 0.8214285714285714, + "655884605/train0": 0.75, + "655884605/train1": 0.42857142857142855, + "655884605/train2": 0.5, + "672328262/train0": 0.7142857142857143, + "672328262/train1": 0.42857142857142855, + "672328262/train2": 0.5357142857142857, + "701667486/train0": 0.6428571428571429, + "701667486/train1": 0.5, + "701667486/train2": 0.6785714285714286, + "747260437/train0": 0.35714285714285715, + "747260437/train1": 0.9285714285714286, + "747260437/train2": 0.5714285714285714, + "750448296/train0": 0.5, + "750448296/train1": 0.3452380952380952, + "750448296/train2": 0.32142857142857145, + "767747424/train0": 0.39285714285714285, + "767747424/train1": 0.46428571428571425, + "767747424/train2": 0.4464285714285714 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5930735865441747, + "low": 0.48606169429698837, + "mean": 0.5421913054265997, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5421913054265997, + "per_task": { + "LiftPegUpright-v1": 0.6304232804232804, + "PickCube-v1": 0.44047619047619047, + "PullCube-v1": 0.5311771561771562, + "PushCube-v1": 0.5866721078485784, + "StackCube-v1": 0.5222077922077923 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5771546391695647, + "low": 0.5128664563739194, + "mean": 0.5450877094160675, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5146408150166321, + "low": 0.5124251320068989, + "mean": 0.5134688196629702, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.5134688196629702, + "per_seed": { + "1073161778/train0": 0.5092955414174704, + "1073161778/train1": 0.5121418326693615, + "1073161778/train2": 0.5109233051180221, + "1113371261/train0": 0.5211029637363088, + "1113371261/train1": 0.5243376283502783, + "1113371261/train2": 0.5258702008499507, + "1151410346/train0": 0.5131067749938627, + "1151410346/train1": 0.5141935806753487, + "1151410346/train2": 0.5140320226786702, + "1261930456/train0": 0.5116618738353578, + "1261930456/train1": 0.5139278716117496, + "1261930456/train2": 0.5121665051198715, + "1324133122/train0": 0.5224112709742389, + "1324133122/train1": 0.5213408711746942, + "1324133122/train2": 0.5254412617086446, + "1324642411/train0": 0.5138598359008081, + "1324642411/train1": 0.516058262207234, + "1324642411/train2": 0.5090482166780169, + "1375492786/train0": 0.505879116784551, + "1375492786/train1": 0.5120529709093987, + "1375492786/train2": 0.5103681394924954, + "141984615/train0": 0.5189149557338043, + "141984615/train1": 0.5139341766826016, + "141984615/train2": 0.505048424591939, + "1461069713/train0": 0.5091120840652484, + "1461069713/train1": 0.5081015084720596, + "1461069713/train2": 0.5120292310110632, + "1480856810/train0": 0.5083154135037583, + "1480856810/train1": 0.5110030837863524, + "1480856810/train2": 0.5118724897465404, + "1509980664/train0": 0.5128660165598996, + "1509980664/train1": 0.511941221071365, + "1509980664/train2": 0.5103616032224382, + "1522272686/train0": 0.5319219008021803, + "1522272686/train1": 0.5176703997119507, + "1522272686/train2": 0.5183770201500114, + "1549324805/train0": 0.5262312992720155, + "1549324805/train1": 0.5173089481291107, + "1549324805/train2": 0.5161816159310763, + "1578990937/train0": 0.5123855862850035, + "1578990937/train1": 0.5156994405981151, + "1578990937/train2": 0.5121377255628292, + "1790188787/train0": 0.5057916478058077, + "1790188787/train1": 0.5062566081726894, + "1790188787/train2": 0.5082783651389248, + "1822210471/train0": 0.5154582541421188, + "1822210471/train1": 0.5122846916174123, + "1822210471/train2": 0.5177920443784734, + "1845489013/train0": 0.5131863087960253, + "1845489013/train1": 0.5137646328948755, + "1845489013/train2": 0.5162114675678457, + "1999022574/train0": 0.5109113095795552, + "1999022574/train1": 0.517373533290843, + "1999022574/train2": 0.5201361575280914, + "2012211287/train0": 0.5077128468462638, + "2012211287/train1": 0.5098868256616201, + "2012211287/train2": 0.5100908012774384, + "2038594191/train0": 0.5257244389989378, + "2038594191/train1": 0.514770372402238, + "2038594191/train2": 0.5169917573441141, + "2054899126/train0": 0.5052995351748154, + "2054899126/train1": 0.5105702026856556, + "2054899126/train2": 0.513192964083336, + "2061635888/train0": 0.5199008764839764, + "2061635888/train1": 0.5189283527559876, + "2061635888/train2": 0.5201168018122745, + "211245842/train0": 0.5086109136222222, + "211245842/train1": 0.507099429830539, + "211245842/train2": 0.5123986202425596, + "278008524/train0": 0.5087559775718801, + "278008524/train1": 0.5071032645290914, + "278008524/train2": 0.5171398278381145, + "307068879/train0": 0.5194284844280975, + "307068879/train1": 0.5225141037494058, + "307068879/train2": 0.518763232568162, + "412102563/train0": 0.5062670117038048, + "412102563/train1": 0.5113786791180399, + "412102563/train2": 0.5089676132647365, + "440577711/train0": 0.5057012649277024, + "440577711/train1": 0.505418210717456, + "441046872/train0": 0.5195701592812082, + "441046872/train1": 0.5215255698164627, + "441046872/train2": 0.5163446864153005, + "509430392/train0": 0.5063024493634376, + "509430392/train1": 0.5086436934457946, + "509430392/train2": 0.5071930884530261, + "585904130/train0": 0.5181208856251349, + "585904130/train1": 0.512435265258071, + "585904130/train2": 0.5094731430629448, + "608016290/train0": 0.5078873058739054, + "608016290/train1": 0.5071201830818504, + "608016290/train2": 0.5096252566953113, + "626486813/train0": 0.5067509279989961, + "626486813/train1": 0.5065516692751414, + "626486813/train2": 0.512420190557138, + "636680137/train0": 0.5055966367137085, + "636680137/train1": 0.5078518602413418, + "636680137/train2": 0.5081316115880623, + "655884605/train0": 0.5166331169260667, + "655884605/train1": 0.5226674723877733, + "655884605/train2": 0.525781459845288, + "672328262/train0": 0.5213462849964556, + "672328262/train1": 0.5156576148233183, + "672328262/train2": 0.5131890723327346, + "701667486/train0": 0.5100395188896416, + "701667486/train1": 0.5074799238651517, + "701667486/train2": 0.5089535245646244, + "747260437/train0": 0.5050963819862415, + "747260437/train1": 0.5172521843522638, + "747260437/train2": 0.5073556110406273, + "750448296/train0": 0.5176625382186207, + "750448296/train1": 0.517466642602347, + "750448296/train2": 0.515094948185793, + "767747424/train0": 0.5123633414032891, + "767747424/train1": 0.5127675241284637, + "767747424/train2": 0.5132197152821629 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5192974399430265, + "low": 0.5115005591575372, + "mean": 0.5148392524815455, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5148392524815455, + "per_task": { + "LiftPegUpright-v1": 0.5135220089933253, + "PickCube-v1": 0.5226564402213808, + "PullCube-v1": 0.5130368946827227, + "PushCube-v1": 0.5094999341420311, + "StackCube-v1": 0.5154809843682674 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5144602751650483, + "low": 0.5125197777585782, + "mean": 0.5134597452924707, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1660720952636879, + "low": 0.12917262621474346, + "mean": 0.14715167772565393, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.14715167772565393, + "per_seed": { + "1073161778/train0": 0.009295541417470354, + "1073161778/train1": 0.02357245304492417, + "1073161778/train2": 0.024790980596263612, + "1113371261/train0": 0.08603989340654827, + "1113371261/train1": 0.13148048549313546, + "1113371261/train2": 0.15270122772147787, + "1151410346/train0": 0.2631067749938627, + "1151410346/train1": 0.3356221521039201, + "1151410346/train2": 0.19260345125009876, + "1261930456/train0": 0.024052411878927926, + "1261930456/train1": 0.15678501446889243, + "1261930456/train2": 0.1306906377372714, + "1324133122/train0": 0.022411270974238895, + "1324133122/train1": 0.08580198596816291, + "1324133122/train2": 0.18884445257706972, + "1324642411/train0": 0.08528840732937953, + "1324642411/train1": 0.10865085479982667, + "1324642411/train2": 0.2131740055442053, + "1375492786/train0": 0.1981868090922433, + "1375492786/train1": 0.02366131480488698, + "1375492786/train2": 0.2753461462217903, + "141984615/train0": 0.4096564728376243, + "141984615/train1": 0.013934176682601551, + "141984615/train2": 0.005048424591938994, + "1461069713/train0": 0.009112084065248438, + "1461069713/train1": 0.3133270629565118, + "1461069713/train2": 0.023685054703222463, + "1480856810/train0": 0.09882744363909879, + "1480856810/train1": 0.15386022664349525, + "1480856810/train2": 0.05955608168203097, + "1509980664/train0": 0.1928660165598996, + "1509980664/train1": 0.008058778928635046, + "1509980664/train2": 0.08179017465100963, + "1522272686/train0": 0.07522095634067683, + "1522272686/train1": 0.2676703997119507, + "1522272686/train2": 0.05409130586429711, + "1549324805/train0": 0.3666258435851274, + "1549324805/train1": 0.26840533758517504, + "1549324805/train2": 0.09096124121178084, + "1578990937/train0": 0.19095701485643207, + "1578990937/train1": 0.23430055940188488, + "1578990937/train2": 0.012137725562829216, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.31563692362276374, + "1790188787/train1": 0.10088624897016774, + "1790188787/train2": 0.3553579984974389, + "1822210471/train0": 0.19882746014359554, + "1822210471/train1": 0.059143879811159095, + "1822210471/train2": 0.12506509847866953, + "1845489013/train0": 0.022527976918260362, + "1845489013/train1": 0.1290925099622674, + "1845489013/train2": 0.30521710386072565, + "1999022574/train0": 0.16766011899187339, + "1999022574/train1": 0.017373533290843013, + "1999022574/train2": 0.1941495567576229, + "2012211287/train0": 0.20657286743945047, + "2012211287/train1": 0.26789095211615765, + "2012211287/train2": 0.38276634157970446, + "2038594191/train0": 0.33141841814391926, + "2038594191/train1": 0.264770372402238, + "2038594191/train2": 0.12413461448697122, + "2054899126/train0": 0.020506910444550552, + "2054899126/train1": 0.09457081562854205, + "2054899126/train2": 0.06483208938690344, + "2061635888/train0": 0.19438483780173788, + "2061635888/train1": 0.15964307581544102, + "2061635888/train2": 0.16297394466941734, + "211245842/train0": 0.06281765780634918, + "211245842/train1": 0.07852800125911047, + "211245842/train2": 0.13045852261458335, + "278008524/train0": 0.013479154071202815, + "278008524/train1": 0.08573408202116861, + "278008524/train2": 0.1514270567762001, + "307068879/train0": 0.016285801286188173, + "307068879/train1": 0.12034303910773714, + "307068879/train2": 0.19552248171755227, + "412102563/train0": 0.18483844027523338, + "412102563/train1": 0.2386213208819601, + "412102563/train2": 0.2232533275504508, + "440577711/train0": 0.2800130207865833, + "440577711/train1": 0.19772590302514825, + "440577711/train2": 0.0, + "441046872/train0": 0.1590012692902204, + "441046872/train1": 0.2284744301835373, + "441046872/train2": 0.09079817072755658, + "509430392/train0": 0.1336975506365624, + "509430392/train1": 0.21862903382693266, + "509430392/train2": 0.08371600245606481, + "585904130/train0": 0.23187911437486508, + "585904130/train1": 0.048149550972356736, + "585904130/train2": 0.2951874287772305, + "608016290/train0": 0.027826979840380295, + "608016290/train1": 0.1499773259389932, + "608016290/train2": 0.16894617187611727, + "626486813/train0": 0.4272928465661017, + "626486813/train1": 0.12300764930449637, + "626486813/train2": 0.14829409515714775, + "636680137/train0": 0.17297479185772013, + "636680137/train1": 0.11499471738419892, + "636680137/train2": 0.3132969598405091, + "655884605/train0": 0.23336688307393327, + "655884605/train1": 0.09409604381634479, + "655884605/train2": 0.025781459845288013, + "672328262/train0": 0.19293942928925867, + "672328262/train1": 0.08708618625188974, + "672328262/train2": 0.022525213381551112, + "701667486/train0": 0.1328176239675013, + "701667486/train1": 0.0074799238651517275, + "701667486/train2": 0.1696179040068042, + "747260437/train0": 0.14795352484338437, + "747260437/train1": 0.41131924421916477, + "747260437/train2": 0.06407296038794408, + "750448296/train0": 0.13781121565484575, + "750448296/train1": 0.17222854736425175, + "750448296/train2": 0.19366637675722168, + "767747424/train0": 0.21687593622230478, + "767747424/train1": 0.09358450213861466, + "767747424/train2": 0.21218415109607414 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.150512999548783, + "low": 0.1364078051516673, + "mean": 0.14321999486997267, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.14321999486997267, + "per_task": { + "LiftPegUpright-v1": 0.1381376537441502, + "PickCube-v1": 0.13232755397230822, + "PullCube-v1": 0.15360683599994118, + "PushCube-v1": 0.1509193203080439, + "StackCube-v1": 0.14110861032541971 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.16681212534814124, + "low": 0.12948847796840834, + "mean": 0.14835202829415556, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 26.716666666666665, + "low": 24.608333333333334, + "mean": 25.747916666666665, + "n": 120, + "num_samples": 1000 + }, + "mean": 25.747916666666665, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 28.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 27.0, + "1324642411/train2": 18.0, + "1375492786/train0": 13.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 28.0, + "1480856810/train1": 28.0, + "1480856810/train2": 28.0, + "1509980664/train0": 25.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 28.0, + "1790188787/train1": 28.0, + "1790188787/train2": 22.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 28.0, + "1999022574/train1": 28.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 27.0, + "2012211287/train2": 28.0, + "2038594191/train0": 28.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 14.0, + "278008524/train1": 14.0, + "278008524/train2": 14.0, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 28.0, + "412102563/train1": 28.0, + "412102563/train2": 28.0, + "440577711/train0": 28.0, + "440577711/train1": 13.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 25.0, + "509430392/train1": 22.0, + "509430392/train2": 22.0, + "585904130/train0": 28.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 28.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 28.0, + "672328262/train1": 28.0, + "672328262/train2": 28.0, + "701667486/train0": 28.0, + "701667486/train1": 28.0, + "701667486/train2": 28.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 28.0, + "750448296/train1": 28.0, + "750448296/train2": 28.0, + "767747424/train0": 22.75, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 27.927272727272726, + "low": 24.69563636363636, + "mean": 26.348969696969696, + "n": 5, + "num_samples": 1000 + }, + "mean": 26.348969696969696, + "per_task": { + "LiftPegUpright-v1": 27.933333333333334, + "PickCube-v1": 28.0, + "PullCube-v1": 24.16, + "PushCube-v1": 23.833333333333332, + "StackCube-v1": 27.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 26.604166666666668, + "low": 24.104166666666668, + "mean": 25.430555555555557, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.18187439026990973, + "low": -0.3196673359572324, + "mean": -0.24804228615917392, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.24804228615917392, + "per_seed": { + "1073161778/train0": -0.05895485055674793, + "1073161778/train1": -0.06517628519498339, + "1073161778/train2": -0.03661228926063653, + "1113371261/train0": -0.04825948731449364, + "1113371261/train1": -0.05766058105545785, + "1113371261/train2": -0.04967133196095555, + "1151410346/train0": -0.06960468013481613, + "1151410346/train1": -0.06131251512798924, + "1151410346/train2": -0.054743002594082435, + "1261930456/train0": -1.042599098629623, + "1261930456/train1": -1.0195808227095515, + "1261930456/train2": -1.0158058232150635, + "1324133122/train0": -0.09201776605615172, + "1324133122/train1": -0.07239433850689977, + "1324133122/train2": -0.05650027981133776, + "1324642411/train0": -1.2895725332761157, + "1324642411/train1": -1.1498268645090013, + "1324642411/train2": -1.1472572709125655, + "1375492786/train0": -0.28547700247176366, + "1375492786/train1": -0.3533321141266833, + "1375492786/train2": -0.38601214633080416, + "141984615/train0": -0.13505606653156377, + "141984615/train1": -0.05316347180765005, + "141984615/train2": -0.10105758631461294, + "1461069713/train0": -0.12922901502057813, + "1461069713/train1": -0.1801597858223893, + "1461069713/train2": -0.02022527141535374, + "1480856810/train0": -1.1191024751801397, + "1480856810/train1": -1.1493088895471595, + "1480856810/train2": -1.0926696735019097, + "1509980664/train0": -0.14698878641541877, + "1509980664/train1": -0.09190610579492059, + "1509980664/train2": -0.1520858341588262, + "1522272686/train0": -0.3891433321605168, + "1522272686/train1": -0.3649738510562886, + "1522272686/train2": -0.35603713675787785, + "1549324805/train0": -0.048325782527749814, + "1549324805/train1": 0.06964181922005985, + "1549324805/train2": -0.0581167593221707, + "1578990937/train0": -0.07399457979836796, + "1578990937/train1": -0.06039227699080513, + "1578990937/train2": -0.09822142936424738, + "1747178030/train0": -0.3331167533441346, + "1747178030/train1": -0.38909781967835777, + "1747178030/train2": -0.46100102368298207, + "1790188787/train0": -1.0792771393876592, + "1790188787/train1": -1.0821139696201234, + "1790188787/train2": -1.0426295433413653, + "1822210471/train0": 0.09172847339145515, + "1822210471/train1": 0.11469903634944659, + "1822210471/train2": 0.11123534194819423, + "1845489013/train0": 0.056123788714222106, + "1845489013/train1": 0.10685267161815218, + "1845489013/train2": 0.11053198394490094, + "1999022574/train0": -0.11814581215768769, + "1999022574/train1": -0.08196834306253598, + "1999022574/train2": -0.09926036807855881, + "2012211287/train0": -0.15769274322565577, + "2012211287/train1": -0.12821380008018857, + "2012211287/train2": -0.09296986205333337, + "2038594191/train0": 0.18263322741939308, + "2038594191/train1": 0.1571311927054344, + "2038594191/train2": 0.1499261458574031, + "2054899126/train0": -0.3176949282327354, + "2054899126/train1": -0.2252511010407856, + "2054899126/train2": -0.40358682075573915, + "2061635888/train0": -0.03336441156547287, + "2061635888/train1": -0.04191814555358769, + "2061635888/train2": -0.033653298036953966, + "211245842/train0": -0.133838155897241, + "211245842/train1": -0.14019095394361425, + "211245842/train2": -0.0967227357905851, + "278008524/train0": -0.07549504506362273, + "278008524/train1": -0.007193800112369225, + "278008524/train2": -0.01000265227293061, + "307068879/train0": 0.031234880094876627, + "307068879/train1": 0.037469066617763, + "307068879/train2": 0.030969288861405195, + "412102563/train0": -0.16113806347993082, + "412102563/train1": -0.15811820951338246, + "412102563/train2": -0.18531182086534548, + "440577711/train0": -1.0254871904912983, + "440577711/train1": -1.067727630841523, + "440577711/train2": -0.9797113666967447, + "441046872/train0": 0.02945650465411806, + "441046872/train1": 0.05890578182706513, + "441046872/train2": 0.017015444441914704, + "509430392/train0": -1.155476122058963, + "509430392/train1": -1.106462385735648, + "509430392/train2": -1.0541420784393922, + "585904130/train0": 0.03414964989475239, + "585904130/train1": -0.027362083160849926, + "585904130/train2": 0.026060848101647938, + "608016290/train0": -0.04985292651651213, + "608016290/train1": -0.05236386089192557, + "608016290/train2": -0.012997582312052308, + "626486813/train0": -0.13103319050538206, + "626486813/train1": -0.07443337753579249, + "626486813/train2": -0.05177776713392784, + "636680137/train0": -0.11026859162288982, + "636680137/train1": -0.14666728076214766, + "636680137/train2": -0.08206888832985737, + "655884605/train0": -0.0781444029185483, + "655884605/train1": -0.01818175818777909, + "655884605/train2": -0.024273861804607055, + "672328262/train0": 0.1294553020456159, + "672328262/train1": 0.09180646718725743, + "672328262/train2": 0.11194999822009422, + "701667486/train0": -0.1980299443706942, + "701667486/train1": -0.1667987048169856, + "701667486/train2": -0.19998664785779915, + "747260437/train0": -0.41293810625522775, + "747260437/train1": -0.43548232619379196, + "747260437/train2": -0.44021912627823445, + "750448296/train0": -0.19117397742857523, + "750448296/train1": -0.08582354011659181, + "750448296/train2": -0.1967269353075053, + "767747424/train0": -0.22610846089363168, + "767747424/train1": -0.20272105535688145, + "767747424/train2": -0.18811356831313295 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.07563776063666727, + "low": -0.7047654778911332, + "mean": -0.34251706800635645, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.34251706800635645, + "per_task": { + "LiftPegUpright-v1": -0.02998554890632619, + "PickCube-v1": -0.3700514399915611, + "PullCube-v1": -0.12701778822001247, + "PushCube-v1": -1.0899306043385468, + "StackCube-v1": -0.09559995857533575 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": -0.17444191322900512, + "low": -0.29324045535978116, + "mean": -0.23513764709132248, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.29583333333333334, + "low": 0.15416666666666667, + "mean": 0.22291666666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.22291666666666668, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40606060606060607, + "low": 0.15903030303030302, + "mean": 0.2754747474747475, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2754747474747475, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.3333333333333333, + "PullCube-v1": 0.17333333333333334, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.18181818181818182 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.2916666666666667, + "low": 0.1597222222222222, + "mean": 0.2222222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7715499536289523, + "low": 0.5113214392965246, + "mean": 0.6364790620334033, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6364790620334033, + "per_seed": { + "1073161778/train0": 0.7180102467536926, + "1073161778/train1": 0.7134289145469666, + "1073161778/train2": 2.0, + "1113371261/train0": 0.0740400105714798, + "1113371261/train1": 0.09622243046760559, + "1113371261/train2": 0.1003647893667221, + "1151410346/train0": 0.5684944987297058, + "1151410346/train1": 0.2439272701740265, + "1151410346/train2": 0.5630103945732117, + "1261930456/train0": 0.09678153693675995, + "1261930456/train1": 0.09997129440307617, + "1261930456/train2": 0.08545441925525665, + "1324133122/train0": 0.0628519058227539, + "1324133122/train1": 0.0843733698129654, + "1324133122/train2": 0.09815146028995514, + "1324642411/train0": 0.21000581979751587, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.14743654429912567, + "1375492786/train2": 0.19452734291553497, + "141984615/train0": 0.023719489574432373, + "141984615/train1": 0.04397805407643318, + "141984615/train2": 0.04447716474533081, + "1461069713/train0": 0.05616961047053337, + "1461069713/train1": 0.05644957348704338, + "1461069713/train2": 0.11542954295873642, + "1480856810/train0": 0.14178013801574707, + "1480856810/train1": 2.0, + "1480856810/train2": 0.19622710347175598, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.83889240026474, + "1522272686/train0": 0.5233199596405029, + "1522272686/train1": 0.5422300696372986, + "1522272686/train2": 2.0, + "1549324805/train0": 0.21791090071201324, + "1549324805/train1": 0.3088321089744568, + "1549324805/train2": 0.21403594315052032, + "1578990937/train0": 0.21425709128379822, + "1578990937/train1": 0.21763037145137787, + "1578990937/train2": 0.23259200155735016, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.1862330436706543, + "1790188787/train2": 2.0, + "1822210471/train0": 0.03802649304270744, + "1822210471/train1": 0.07539409399032593, + "1822210471/train2": 0.058819372206926346, + "1845489013/train0": 0.0428844690322876, + "1845489013/train1": 0.09529292583465576, + "1845489013/train2": 0.0861951932311058, + "1999022574/train0": 0.5970457196235657, + "1999022574/train1": 2.0, + "1999022574/train2": 0.7970014810562134, + "2012211287/train0": 0.7277753949165344, + "2012211287/train1": 2.0, + "2012211287/train2": 0.7270426154136658, + "2038594191/train0": 0.15247176587581635, + "2038594191/train1": 0.1902807205915451, + "2038594191/train2": 0.24880830943584442, + "2054899126/train0": 1.0726270377635956, + "2054899126/train1": 1.0755530372262, + "2054899126/train2": 1.0683744624257088, + "2061635888/train0": 0.07642464339733124, + "2061635888/train1": 0.09046469628810883, + "2061635888/train2": 0.07902413606643677, + "211245842/train0": 0.5822281837463379, + "211245842/train1": 0.2016763538122177, + "211245842/train2": 0.2142850160598755, + "278008524/train0": 1.0722030997276306, + "278008524/train1": 1.1246033310890198, + "278008524/train2": 1.157032236456871, + "307068879/train0": 0.20280277729034424, + "307068879/train1": 0.20222051441669464, + "307068879/train2": 0.2035302072763443, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 0.8275061845779419, + "440577711/train0": 0.2057151049375534, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.203963503241539, + "441046872/train1": 0.20609354972839355, + "441046872/train2": 0.2042645812034607, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 0.16448359191417694, + "585904130/train1": 0.15698926150798798, + "585904130/train2": 0.2938849627971649, + "608016290/train0": 0.6124615669250488, + "608016290/train1": 0.6101007461547852, + "608016290/train2": 0.5705108642578125, + "626486813/train0": 0.05774742551147938, + "626486813/train1": 0.0619367565959692, + "626486813/train2": 0.08006523177027702, + "636680137/train0": 0.10778365284204483, + "636680137/train1": 0.11253269761800766, + "636680137/train2": 0.1247115358710289, + "655884605/train0": 0.15739890933036804, + "655884605/train1": 0.15038728713989258, + "655884605/train2": 0.17242512106895447, + "672328262/train0": 0.05979333445429802, + "672328262/train1": 0.053388457745313644, + "672328262/train2": 0.07216622680425644, + "701667486/train0": 0.7879213094711304, + "701667486/train1": 0.7947455048561096, + "701667486/train2": 2.0, + "747260437/train0": 0.19230616092681885, + "747260437/train1": 0.1342230886220932, + "747260437/train2": 0.18527252972126007, + "750448296/train0": 0.20943730572859445, + "750448296/train1": 0.22532296180725098, + "750448296/train2": 0.2481476068496704, + "767747424/train0": 0.5789013803005219, + "767747424/train1": 0.5825987188145518, + "767747424/train2": 0.5809952197596431 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.021850009759267, + "low": 0.5703265610120514, + "mean": 0.7913498574552512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7913498574552512, + "per_task": { + "LiftPegUpright-v1": 0.6601470321416855, + "PickCube-v1": 1.021850009759267, + "PullCube-v1": 0.46170220340291657, + "PushCube-v1": 1.1790093589160178, + "StackCube-v1": 0.634040683056369 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 0.7532538059684966, + "low": 0.5049100180880891, + "mean": 0.6232009150553495, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 3.4291666666666667, + "low": 2.6791666666666667, + "mean": 3.051388888888889, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.051388888888889, + "per_seed": { + "1073161778/train0": 3.0, + "1073161778/train1": 6.0, + "1073161778/train2": 5.0, + "1113371261/train0": 0.0, + "1113371261/train1": 4.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 3.0, + "1151410346/train2": 5.0, + "1261930456/train0": 7.0, + "1261930456/train1": 4.0, + "1261930456/train2": 3.0, + "1324133122/train0": 4.0, + "1324133122/train1": 7.0, + "1324133122/train2": 4.0, + "1324642411/train0": 7.0, + "1324642411/train1": 0.0, + "1324642411/train2": 3.0, + "1375492786/train0": 1.0, + "1375492786/train1": 3.0, + "1375492786/train2": 2.0, + "141984615/train0": 7.0, + "141984615/train1": 1.0, + "141984615/train2": 3.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 3.0, + "1509980664/train0": 0.0, + "1509980664/train1": 4.0, + "1509980664/train2": 4.0, + "1522272686/train0": 7.0, + "1522272686/train1": 6.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 4.0, + "1578990937/train1": 4.0, + "1578990937/train2": 5.0, + "1747178030/train0": 1.0, + "1747178030/train1": 4.0, + "1747178030/train2": 6.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 2.0, + "1822210471/train0": 5.0, + "1822210471/train1": 5.0, + "1822210471/train2": 3.0, + "1845489013/train0": 2.0, + "1845489013/train1": 6.0, + "1845489013/train2": 3.0, + "1999022574/train0": 1.0, + "1999022574/train1": 5.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 4.0, + "2038594191/train0": 7.0, + "2038594191/train1": 4.0, + "2038594191/train2": 2.0, + "2054899126/train0": 3.0, + "2054899126/train1": 2.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 4.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 5.0, + "211245842/train2": 2.0, + "278008524/train0": 5.0, + "278008524/train1": 3.0, + "278008524/train2": 1.0, + "307068879/train0": 6.0, + "307068879/train1": 1.0, + "307068879/train2": 4.0, + "412102563/train0": 0.0, + "412102563/train1": 3.0, + "412102563/train2": 3.0, + "440577711/train0": 1.0, + "440577711/train1": 6.0, + "440577711/train2": 0.0, + "441046872/train0": 7.0, + "441046872/train1": 7.0, + "441046872/train2": 5.0, + "509430392/train0": 3.0, + "509430392/train1": 5.0, + "509430392/train2": 1.0, + "585904130/train0": 6.0, + "585904130/train1": 3.0, + "585904130/train2": 1.0, + "608016290/train0": 6.0, + "608016290/train1": 3.0, + "608016290/train2": 3.0, + "626486813/train0": 2.5, + "626486813/train1": 0.5, + "626486813/train2": 2.0, + "636680137/train0": 6.0, + "636680137/train1": 6.0, + "636680137/train2": 1.0, + "655884605/train0": 3.0, + "655884605/train1": 6.0, + "655884605/train2": 4.0, + "672328262/train0": 2.0, + "672328262/train1": 5.0, + "672328262/train2": 7.0, + "701667486/train0": 1.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 0.0, + "747260437/train2": 2.0, + "750448296/train0": 2.0, + "750448296/train1": 3.0, + "750448296/train2": 1.6666666666666667, + "767747424/train0": 2.0, + "767747424/train1": 1.25, + "767747424/train2": 3.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 4.1786666666666665, + "low": 2.6728888888888886, + "mean": 3.423111111111111, + "n": 5, + "num_samples": 1000 + }, + "mean": 3.423111111111111, + "per_task": { + "LiftPegUpright-v1": 4.266666666666667, + "PickCube-v1": 4.666666666666667, + "PullCube-v1": 2.6266666666666665, + "PushCube-v1": 2.5555555555555554, + "StackCube-v1": 3.0 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.2430555555555554, + "low": 2.5902777777777777, + "mean": 2.9166666666666665, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3958333333333333, + "low": 0.2277777777777778, + "mean": 0.3083333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3083333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8053333333333332, + "low": 0.10836363636363637, + "mean": 0.45684848484848484, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.45684848484848484, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.4097222222222222, + "low": 0.2569444444444444, + "mean": 0.3333333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": 0.0, + "mean": 0.0, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.0, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": 0.0, + "mean": 0.0, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.0, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.0, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.0, + "low": 0.0, + "mean": 0.0, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9544915752174954, + "low": 0.65628542716796, + "mean": 0.798868260888331, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.798868260888331, + "per_seed": { + "1073161778/train0": 0.7119815349578857, + "1073161778/train1": 0.713421642780304, + "1073161778/train2": 0.7243968844413757, + "1113371261/train0": 0.0680118203163147, + "1113371261/train1": 0.08607026934623718, + "1113371261/train2": 0.09928342700004578, + "1151410346/train0": 0.23053553700447083, + "1151410346/train1": 0.2249394953250885, + "1151410346/train2": 0.23354609310626984, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.06854432821273804, + "1324133122/train2": 0.09103855490684509, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.04150080680847168, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.09350365400314331, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.21791090071201324, + "1549324805/train1": 0.3088321089744568, + "1549324805/train2": 0.12139803171157837, + "1578990937/train0": 0.21093030273914337, + "1578990937/train1": 0.21256697177886963, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.03715964034199715, + "1822210471/train1": 0.06487039476633072, + "1822210471/train2": 0.057336073368787766, + "1845489013/train0": 0.04260633513331413, + "1845489013/train1": 0.06991276890039444, + "1845489013/train2": 0.0861951932311058, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.15247176587581635, + "2038594191/train1": 0.14092111587524414, + "2038594191/train2": 0.10970946401357651, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.07517768442630768, + "2061635888/train1": 0.08908586204051971, + "2061635888/train2": 0.07194386422634125, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.0467599928379059, + "278008524/train1": 1.1101078614592552, + "278008524/train2": 1.115786612033844, + "307068879/train0": 0.19984664022922516, + "307068879/train1": 0.20222051441669464, + "307068879/train2": 0.2035302072763443, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.203963503241539, + "441046872/train1": 0.20609354972839355, + "441046872/train2": 0.20165641605854034, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.14155812561511993, + "585904130/train1": 0.12263611704111099, + "585904130/train2": 0.13704223930835724, + "608016290/train0": 0.5531623363494873, + "608016290/train1": 0.5450301170349121, + "608016290/train2": 0.4804247319698334, + "626486813/train0": 0.03338620066642761, + "626486813/train1": 0.043074944987893105, + "626486813/train2": 0.04263448715209961, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.12485501170158386, + "655884605/train2": 0.16552868485450745, + "672328262/train0": 0.045009810477495193, + "672328262/train1": 0.05302006006240845, + "672328262/train2": 0.0664752721786499, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.8171049058437347, + "750448296/train1": 0.7914677709341049, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0747087560594082, + "767747424/train1": 1.056986487004906, + "767747424/train2": 1.0559786520898342 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6340942325465604, + "low": 0.5486931356087779, + "mean": 1.0889083013954488, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.0889083013954488, + "per_task": { + "LiftPegUpright-v1": 0.475364543000857, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7987058012435834, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.4970153953992959 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.9662950354912836, + "low": 0.6878012982197106, + "mean": 0.830257743739316, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.0015038774090094696, + "low": -0.009511410182393673, + "mean": -0.003935215297517263, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.003935215297517263, + "per_seed": { + "1073161778/train0": -0.013536453247070312, + "1073161778/train1": -0.0120963454246521, + "1073161778/train2": -0.0011211037635803223, + "1113371261/train0": -0.043087467551231384, + "1113371261/train1": -0.0250290185213089, + "1113371261/train2": -0.011815860867500305, + "1151410346/train0": 0.0020620226860046387, + "1151410346/train1": -0.0035340189933776855, + "1151410346/train2": 0.00507257878780365, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": -0.015682727098464966, + "1324133122/train2": 0.00681149959564209, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": -0.12633192539215088, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": -0.13502393662929535, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.04541558027267456, + "1549324805/train1": 0.1363367885351181, + "1549324805/train2": -0.051097288727760315, + "1578990937/train0": -0.0033356696367263794, + "1578990937/train1": -0.001699000597000122, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": -0.05726582184433937, + "1822210471/train1": -0.0295550674200058, + "1822210471/train2": -0.03708938881754875, + "1845489013/train0": -0.06995167210698128, + "1845489013/train1": -0.04264523833990097, + "1845489013/train2": -0.026362814009189606, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0574311688542366, + "2038594191/train1": 0.0458805188536644, + "2038594191/train2": 0.014668866991996765, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": -0.011029243469238281, + "2061635888/train1": 0.002878934144973755, + "2061635888/train2": -0.014263063669204712, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": -0.07824721932411194, + "278008524/train1": -0.014899350702762604, + "278008524/train2": -0.009220600128173828, + "307068879/train0": -0.009305715560913086, + "307068879/train1": -0.0069318413734436035, + "307068879/train2": -0.005622148513793945, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": -0.005804136395454407, + "441046872/train1": -0.0036740899085998535, + "441046872/train2": -0.008111223578453064, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.07052158564329147, + "585904130/train1": 0.05159957706928253, + "585904130/train2": 0.06600569933652878, + "608016290/train0": 0.027802228927612305, + "608016290/train1": 0.01967000961303711, + "608016290/train2": -0.044935375452041626, + "626486813/train0": -0.00835469365119934, + "626486813/train1": 0.0013340506702661514, + "626486813/train2": 0.0008935928344726562, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": -0.01922038197517395, + "655884605/train2": 0.021453291177749634, + "672328262/train0": -0.060924429446458817, + "672328262/train1": -0.05291417986154556, + "672328262/train2": -0.03945896774530411, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.056601578990618386, + "750448296/train1": 0.030964444080988567, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": -0.01772226905450225, + "767747424/train2": -0.018730103969573975 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.0006692007635578964, + "low": -0.004184890265717651, + "mean": -0.0024270455146377734, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.0024270455146377734, + "per_task": { + "LiftPegUpright-v1": -0.004244412978490194, + "PickCube-v1": 0.0, + "PullCube-v1": -0.004544810776909192, + "PushCube-v1": 0.0, + "StackCube-v1": -0.003346003817789482 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.002463926393021312, + "low": -0.009235467632404633, + "mean": -0.003576007839809689, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.20784406180415924, + "low": 0.06403497271110407, + "mean": 0.12993349621408723, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.12993349621408723, + "per_seed": { + "1073161778/train0": 0.006028711795806885, + "1073161778/train1": 7.271766662597656e-06, + "1073161778/train2": 1.2756031155586243, + "1113371261/train0": 0.0060281902551651, + "1113371261/train1": 0.010152161121368408, + "1113371261/train2": 0.0010813623666763306, + "1151410346/train0": 0.337958961725235, + "1151410346/train1": 0.01898777484893799, + "1151410346/train2": 0.32946430146694183, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.015829041600227356, + "1324133122/train2": 0.007112905383110046, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.002477247267961502, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.02192588895559311, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.167920172214508, + "1509980664/train1": 1.167920172214508, + "1509980664/train2": 0.006812572479248047, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.09263791143894196, + "1578990937/train0": 0.003326788544654846, + "1578990937/train1": 0.00506339967250824, + "1578990937/train2": 0.018326029181480408, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0008668527007102966, + "1822210471/train1": 0.010523699223995209, + "1822210471/train2": 0.0014832988381385803, + "1845489013/train0": 0.00027813389897346497, + "1845489013/train1": 0.025380156934261322, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.1720311045646667, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.2717533111572266, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.049359604716300964, + "2038594191/train2": 0.1390988454222679, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0012469589710235596, + "2061635888/train1": 0.0013788342475891113, + "2061635888/train2": 0.00708027184009552, + "211245842/train0": 0.3979237377643585, + "211245842/train1": 0.017371907830238342, + "211245842/train2": 0.029980570077896118, + "278008524/train0": 0.02544310688972473, + "278008524/train1": 0.014495469629764557, + "278008524/train2": 0.04124562442302704, + "307068879/train0": 0.0029561370611190796, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.002608165144920349, + "509430392/train0": 1.9592653959989548, + "509430392/train1": 1.9592653959989548, + "509430392/train2": 1.9592653959989548, + "585904130/train0": 0.022925466299057007, + "585904130/train1": 0.034353144466876984, + "585904130/train2": 0.15684272348880768, + "608016290/train0": 0.05929923057556152, + "608016290/train1": 0.06507062911987305, + "608016290/train2": 0.09008613228797913, + "626486813/train0": 0.024361224845051765, + "626486813/train1": 0.018861811608076096, + "626486813/train2": 0.037430744618177414, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.005137942731380463, + "655884605/train0": 0.01332351565361023, + "655884605/train1": 0.025532275438308716, + "655884605/train2": 0.0068964362144470215, + "672328262/train0": 0.014783523976802826, + "672328262/train1": 0.00036839768290519714, + "672328262/train2": 0.005690954625606537, + "701667486/train0": 0.0, + "701667486/train1": 0.0034714341163635254, + "701667486/train2": 1.208725929260254, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.006840065121650696, + "750448296/train1": 0.03293752670288086, + "750448296/train2": 0.11175449689229329, + "767747424/train0": 0.019455675035715103, + "767747424/train1": 0.022256564814597368, + "767747424/train2": 0.020647740922868252 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.24082352766484924, + "low": 0.04127709191540877, + "mean": 0.1428341009278189, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.1428341009278189, + "per_task": { + "LiftPegUpright-v1": 0.18489418029785157, + "PickCube-v1": 0.0, + "PullCube-v1": 0.02149127927919229, + "PushCube-v1": 0.32654423266649246, + "StackCube-v1": 0.18124081239555823 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.17804985040695304, + "low": 0.05555807914222694, + "mean": 0.11280556699623251, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.125, + "low": 0.03333333333333333, + "mean": 0.075, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.075, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.14181818181818182, + "low": 0.024242424242424242, + "mean": 0.08424242424242424, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.08424242424242424, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.10416666666666667, + "low": 0.027777777777777776, + "mean": 0.0625, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6597222222222222, + "low": 0.4847222222222222, + "mean": 0.5708333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5708333333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8072727272727273, + "low": 0.40533333333333327, + "mean": 0.6063030303030303, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6063030303030303, + "per_task": { + "LiftPegUpright-v1": 0.8666666666666667, + "PickCube-v1": 0.6666666666666666, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.8181818181818182 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.5972222222222222, + "low": 0.4375, + "mean": 0.5138888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1496533814558965, + "low": 0.8442967605000982, + "mean": 0.9982878092624661, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.9982878092624661, + "per_seed": { + "1073161778/train0": 1.2819897532463074, + "1073161778/train1": 1.2865710854530334, + "1073161778/train2": 0.0, + "1113371261/train0": 1.9259599894285202, + "1113371261/train1": 1.9037775695323944, + "1113371261/train2": 1.899635210633278, + "1151410346/train0": 1.4315055012702942, + "1151410346/train1": 1.7560727298259735, + "1151410346/train2": 1.4369896054267883, + "1261930456/train0": 1.90321846306324, + "1261930456/train1": 1.9000287055969238, + "1261930456/train2": 1.9145455807447433, + "1324133122/train0": 1.937148094177246, + "1324133122/train1": 1.9156266301870346, + "1324133122/train2": 1.9018485397100449, + "1324642411/train0": 1.7899941802024841, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.8525634557008743, + "1375492786/train2": 1.805472657084465, + "141984615/train0": 0.2620577812194824, + "141984615/train1": 0.2417992167174816, + "141984615/train2": 0.24130010604858398, + "1461069713/train0": 0.10317284241318703, + "1461069713/train1": 0.10289287939667702, + "1461069713/train2": 0.04391290992498398, + "1480856810/train0": 0.06770022213459015, + "1480856810/train1": 0.0, + "1480856810/train2": 0.013253256678581238, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.16110759973526, + "1522272686/train0": 1.476680040359497, + "1522272686/train1": 1.4577699303627014, + "1522272686/train2": 0.0, + "1549324805/train0": 0.07954518496990204, + "1549324805/train1": 0.0, + "1549324805/train2": 0.08342014253139496, + "1578990937/train0": 1.7857429087162018, + "1578990937/train1": 1.7823696285486221, + "1578990937/train2": 1.7674079984426498, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8137669563293457, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9619735069572926, + "1822210471/train1": 1.924605906009674, + "1822210471/train2": 1.9411806277930737, + "1845489013/train0": 1.9571155309677124, + "1845489013/train1": 1.9047070741653442, + "1845489013/train2": 1.9138048067688942, + "1999022574/train0": 1.4029542803764343, + "1999022574/train1": 0.0, + "1999022574/train2": 1.2029985189437866, + "2012211287/train0": 1.2722246050834656, + "2012211287/train1": 0.0, + "2012211287/train2": 1.2729573845863342, + "2038594191/train0": 1.8475282341241837, + "2038594191/train1": 1.809719279408455, + "2038594191/train2": 1.7511916905641556, + "2054899126/train0": 0.9273729622364044, + "2054899126/train1": 0.9244469627737999, + "2054899126/train2": 0.9316255375742912, + "2061635888/train0": 1.9235753566026688, + "2061635888/train1": 1.9095353037118912, + "2061635888/train2": 1.9209758639335632, + "211245842/train0": 1.417771816253662, + "211245842/train1": 1.7983236461877823, + "211245842/train2": 1.7857149839401245, + "278008524/train0": 0.04170554131269455, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.7971972227096558, + "307068879/train1": 1.7977794855833054, + "307068879/train2": 1.7964697927236557, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.172493815422058, + "440577711/train0": 1.7942848950624466, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.796036496758461, + "441046872/train1": 1.7939064502716064, + "441046872/train2": 1.7957354187965393, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.835516408085823, + "585904130/train1": 1.843010738492012, + "585904130/train2": 1.706115037202835, + "608016290/train0": 1.3875384330749512, + "608016290/train1": 1.3898992538452148, + "608016290/train2": 1.4294891357421875, + "626486813/train0": 0.1078349556773901, + "626486813/train1": 0.10364562459290028, + "626486813/train2": 0.08551714941859245, + "636680137/train0": 0.06278067082166672, + "636680137/train1": 0.05803162604570389, + "636680137/train2": 0.04585278779268265, + "655884605/train0": 1.842601090669632, + "655884605/train1": 1.8496127128601074, + "655884605/train2": 1.8275748789310455, + "672328262/train0": 1.940206665545702, + "672328262/train1": 1.9466115422546864, + "672328262/train2": 1.9278337731957436, + "701667486/train0": 1.2120786905288696, + "701667486/train1": 1.2052544951438904, + "701667486/train2": 0.0, + "747260437/train0": 0.08635708689689636, + "747260437/train1": 0.144440159201622, + "747260437/train2": 0.09339071810245514, + "750448296/train0": 1.1985591898361843, + "750448296/train1": 1.1875135501225789, + "750448296/train2": 1.1598488887151082, + "767747424/train0": 0.10475751012563705, + "767747424/train1": 0.1008116127923131, + "767747424/train2": 0.09709237236529589 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2779549119689249, + "low": 0.7197053071856498, + "mean": 0.9988301095772872, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9988301095772872, + "per_task": { + "LiftPegUpright-v1": 1.3398529678583144, + "PickCube-v1": 0.9781499902407328, + "PullCube-v1": 0.6881442584097386, + "PushCube-v1": 0.6220440144340197, + "StackCube-v1": 1.365959316943631 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 1.045588894850678, + "low": 0.7773104406013671, + "mean": 0.9091424554741631, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.07465520951292605, + "low": -0.07465520951292605, + "mean": -0.07465520951292605, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.07465520951292605, + "per_seed": { + "1073161778/train0": -0.07465520951292609, + "1073161778/train1": -0.07465520951292609, + "1073161778/train2": -0.07465520951292609, + "1113371261/train0": -0.07465520951292609, + "1113371261/train1": -0.07465520951292609, + "1113371261/train2": -0.07465520951292609, + "1151410346/train0": -0.07465520951292609, + "1151410346/train1": -0.07465520951292609, + "1151410346/train2": -0.07465520951292609, + "1261930456/train0": -0.07465520951292609, + "1261930456/train1": -0.07465520951292609, + "1261930456/train2": -0.07465520951292609, + "1324133122/train0": -0.07465520951292609, + "1324133122/train1": -0.07465520951292609, + "1324133122/train2": -0.07465520951292609, + "1324642411/train0": -0.07465520951292609, + "1324642411/train1": -0.07465520951292609, + "1324642411/train2": -0.07465520951292609, + "1375492786/train0": -0.07465520951292609, + "1375492786/train1": -0.07465520951292609, + "1375492786/train2": -0.07465520951292609, + "141984615/train0": -0.07465520951292609, + "141984615/train1": -0.07465520951292609, + "141984615/train2": -0.07465520951292609, + "1461069713/train0": -0.07465520951292609, + "1461069713/train1": -0.07465520951292609, + "1461069713/train2": -0.07465520951292609, + "1480856810/train0": -0.07465520951292609, + "1480856810/train1": -0.07465520951292609, + "1480856810/train2": -0.07465520951292609, + "1509980664/train0": -0.07465520951292609, + "1509980664/train1": -0.07465520951292609, + "1509980664/train2": -0.07465520951292609, + "1522272686/train0": -0.07465520951292609, + "1522272686/train1": -0.07465520951292609, + "1522272686/train2": -0.07465520951292609, + "1549324805/train0": -0.07465520951292609, + "1549324805/train1": -0.07465520951292609, + "1549324805/train2": -0.07465520951292609, + "1578990937/train0": -0.07465520951292609, + "1578990937/train1": -0.07465520951292609, + "1578990937/train2": -0.07465520951292609, + "1747178030/train0": -0.07465520951292609, + "1747178030/train1": -0.07465520951292609, + "1747178030/train2": -0.07465520951292609, + "1790188787/train0": -0.07465520951292609, + "1790188787/train1": -0.07465520951292609, + "1790188787/train2": -0.07465520951292609, + "1822210471/train0": -0.07465520951292609, + "1822210471/train1": -0.07465520951292609, + "1822210471/train2": -0.07465520951292609, + "1845489013/train0": -0.07465520951292609, + "1845489013/train1": -0.07465520951292609, + "1845489013/train2": -0.07465520951292609, + "1999022574/train0": -0.07465520951292609, + "1999022574/train1": -0.07465520951292609, + "1999022574/train2": -0.07465520951292609, + "2012211287/train0": -0.07465520951292609, + "2012211287/train1": -0.07465520951292609, + "2012211287/train2": -0.07465520951292609, + "2038594191/train0": -0.07465520951292609, + "2038594191/train1": -0.07465520951292609, + "2038594191/train2": -0.07465520951292609, + "2054899126/train0": -0.07465520951292609, + "2054899126/train1": -0.07465520951292609, + "2054899126/train2": -0.07465520951292609, + "2061635888/train0": -0.07465520951292609, + "2061635888/train1": -0.07465520951292609, + "2061635888/train2": -0.07465520951292609, + "211245842/train0": -0.07465520951292609, + "211245842/train1": -0.07465520951292609, + "211245842/train2": -0.07465520951292609, + "278008524/train0": -0.07465520951292609, + "278008524/train1": -0.07465520951292609, + "278008524/train2": -0.07465520951292609, + "307068879/train0": -0.07465520951292609, + "307068879/train1": -0.07465520951292609, + "307068879/train2": -0.07465520951292609, + "412102563/train0": -0.07465520951292609, + "412102563/train1": -0.07465520951292609, + "412102563/train2": -0.07465520951292609, + "440577711/train0": -0.07465520951292609, + "440577711/train1": -0.07465520951292609, + "440577711/train2": -0.07465520951292609, + "441046872/train0": -0.07465520951292609, + "441046872/train1": -0.07465520951292609, + "441046872/train2": -0.07465520951292609, + "509430392/train0": -0.07465520951292609, + "509430392/train1": -0.07465520951292609, + "509430392/train2": -0.07465520951292609, + "585904130/train0": -0.07465520951292609, + "585904130/train1": -0.07465520951292609, + "585904130/train2": -0.07465520951292609, + "608016290/train0": -0.07465520951292609, + "608016290/train1": -0.07465520951292609, + "608016290/train2": -0.07465520951292609, + "626486813/train0": -0.07465520951292609, + "626486813/train1": -0.07465520951292609, + "626486813/train2": -0.07465520951292609, + "636680137/train0": -0.07465520951292609, + "636680137/train1": -0.07465520951292609, + "636680137/train2": -0.07465520951292609, + "655884605/train0": -0.07465520951292609, + "655884605/train1": -0.07465520951292609, + "655884605/train2": -0.07465520951292609, + "672328262/train0": -0.07465520951292609, + "672328262/train1": -0.07465520951292609, + "672328262/train2": -0.07465520951292609, + "701667486/train0": -0.07465520951292609, + "701667486/train1": -0.07465520951292609, + "701667486/train2": -0.07465520951292609, + "747260437/train0": -0.07465520951292609, + "747260437/train1": -0.07465520951292609, + "747260437/train2": -0.07465520951292609, + "750448296/train0": -0.07465520951292609, + "750448296/train1": -0.07465520951292609, + "750448296/train2": -0.07465520951292609, + "767747424/train0": -0.07465520951292609, + "767747424/train1": -0.07465520951292609, + "767747424/train2": -0.07465520951292609 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.07465520951292605, + "low": -0.07465520951292608, + "mean": -0.07465520951292606, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.07465520951292606, + "per_task": { + "LiftPegUpright-v1": -0.07465520951292606, + "PickCube-v1": -0.07465520951292609, + "PullCube-v1": -0.07465520951292605, + "PushCube-v1": -0.07465520951292606, + "StackCube-v1": -0.07465520951292605 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": -0.07465520951292605, + "low": -0.07465520951292605, + "mean": -0.07465520951292605, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": -0.07465520951292609, + "threshold_scope": "global", + "weights": [ + -0.36334951815701444, + 0.015404318964937977, + -0.026482870827477342, + 0.0177984924307096, + 0.006390815196555558, + -0.005178278892529562, + 0.04655132428547662, + -0.2670085784734563, + 0.12600470489027654, + 2.1281155427473213e-17, + -0.026718723572960307, + -0.018021813800449968, + -0.027998812454883754, + 0.0409502756879879, + -0.0030531847411340886, + 0.045775166189357154, + 0.03255060539976545, + -0.018952518681638813, + -0.026718723572960464, + -0.018021813800450193, + -0.027998812454883705, + 0.040950275687988084, + -0.0030531847411340157, + 0.04577516618935713, + 0.03255060539976545, + -0.01895251868163879, + -0.01101011900411407, + -0.0033706883847928627, + -0.003545399682168754, + 0.013835278265787822, + -6.815023064737219e-05, + 0.011020435162868873, + 0.04148023174569504, + -0.01669876117536586, + 0.0, + 0.015824117068425295, + 0.0317994314196069 + ] +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..e23ab09fca136fca0225be05f3564b46074759fd --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics_by_seed.json @@ -0,0 +1,3234 @@ +{ + "1073161778/train0": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5092955414174704, + "pairwise_causal_calibration_ece": 0.009295541417470354, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05895485055674793, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7119815349578857, + "selected_utility_gain_over_base": -0.013536453247070312, + "selector_regret": 0.006028711795806885, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "tau": -0.07465520951292609 + }, + "1073161778/train1": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5121418326693615, + "pairwise_causal_calibration_ece": 0.02357245304492417, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06517628519498339, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.713421642780304, + "selected_utility_gain_over_base": -0.0120963454246521, + "selector_regret": 7.271766662597656e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "tau": -0.07465520951292609 + }, + "1073161778/train2": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5109233051180221, + "pairwise_causal_calibration_ece": 0.024790980596263612, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03661228926063653, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7243968844413757, + "selected_utility_gain_over_base": -0.0011211037635803223, + "selector_regret": 1.2756031155586243, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5211029637363088, + "pairwise_causal_calibration_ece": 0.08603989340654827, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04825948731449364, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0680118203163147, + "selected_utility_gain_over_base": -0.043087467551231384, + "selector_regret": 0.0060281902551651, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "tau": -0.07465520951292609 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5243376283502783, + "pairwise_causal_calibration_ece": 0.13148048549313546, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05766058105545785, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08607026934623718, + "selected_utility_gain_over_base": -0.0250290185213089, + "selector_regret": 0.010152161121368408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "tau": -0.07465520951292609 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5258702008499507, + "pairwise_causal_calibration_ece": 0.15270122772147787, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04967133196095555, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09928342700004578, + "selected_utility_gain_over_base": -0.011815860867500305, + "selector_regret": 0.0010813623666763306, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "tau": -0.07465520951292609 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5131067749938627, + "pairwise_causal_calibration_ece": 0.2631067749938627, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06960468013481613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.23053553700447083, + "selected_utility_gain_over_base": 0.0020620226860046387, + "selector_regret": 0.337958961725235, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "tau": -0.07465520951292609 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5141935806753487, + "pairwise_causal_calibration_ece": 0.3356221521039201, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06131251512798924, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2249394953250885, + "selected_utility_gain_over_base": -0.0035340189933776855, + "selector_regret": 0.01898777484893799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "tau": -0.07465520951292609 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5140320226786702, + "pairwise_causal_calibration_ece": 0.19260345125009876, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.054743002594082435, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.23354609310626984, + "selected_utility_gain_over_base": 0.00507257878780365, + "selector_regret": 0.32946430146694183, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "tau": -0.07465520951292609 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5116618738353578, + "pairwise_causal_calibration_ece": 0.024052411878927926, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.042599098629623, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "tau": -0.07465520951292609 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5139278716117496, + "pairwise_causal_calibration_ece": 0.15678501446889243, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0195808227095515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "tau": -0.07465520951292609 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5121665051198715, + "pairwise_causal_calibration_ece": 0.1306906377372714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0158058232150635, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "tau": -0.07465520951292609 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5224112709742389, + "pairwise_causal_calibration_ece": 0.022411270974238895, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09201776605615172, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "tau": -0.07465520951292609 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5213408711746942, + "pairwise_causal_calibration_ece": 0.08580198596816291, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07239433850689977, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06854432821273804, + "selected_utility_gain_over_base": -0.015682727098464966, + "selector_regret": 0.015829041600227356, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "tau": -0.07465520951292609 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5254412617086446, + "pairwise_causal_calibration_ece": 0.18884445257706972, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05650027981133776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09103855490684509, + "selected_utility_gain_over_base": 0.00681149959564209, + "selector_regret": 0.007112905383110046, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "tau": -0.07465520951292609 + }, + "1324642411/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5138598359008081, + "pairwise_causal_calibration_ece": 0.08528840732937953, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.2895725332761157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "tau": -0.07465520951292609 + }, + "1324642411/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4074074074074074, + "pairwise_causal_calibration_confidence": 0.516058262207234, + "pairwise_causal_calibration_ece": 0.10865085479982667, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": -1.1498268645090013, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1324642411/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7222222222222222, + "pairwise_causal_calibration_confidence": 0.5090482166780169, + "pairwise_causal_calibration_ece": 0.2131740055442053, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": -1.1472572709125655, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.505879116784551, + "pairwise_causal_calibration_ece": 0.1981868090922433, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.28547700247176366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5120529709093987, + "pairwise_causal_calibration_ece": 0.02366131480488698, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3533321141266833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "tau": -0.07465520951292609 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5103681394924954, + "pairwise_causal_calibration_ece": 0.2753461462217903, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.38601214633080416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "tau": -0.07465520951292609 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.5189149557338043, + "pairwise_causal_calibration_ece": 0.4096564728376243, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.13505606653156377, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "tau": -0.07465520951292609 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5139341766826016, + "pairwise_causal_calibration_ece": 0.013934176682601551, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05316347180765005, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04150080680847168, + "selected_utility_gain_over_base": -0.12633192539215088, + "selector_regret": 0.002477247267961502, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "tau": -0.07465520951292609 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.505048424591939, + "pairwise_causal_calibration_ece": 0.005048424591938994, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.10105758631461294, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "tau": -0.07465520951292609 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5091120840652484, + "pairwise_causal_calibration_ece": 0.009112084065248438, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.12922901502057813, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "tau": -0.07465520951292609 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5081015084720596, + "pairwise_causal_calibration_ece": 0.3133270629565118, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1801597858223893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "tau": -0.07465520951292609 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5120292310110632, + "pairwise_causal_calibration_ece": 0.023685054703222463, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.02022527141535374, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09350365400314331, + "selected_utility_gain_over_base": -0.13502393662929535, + "selector_regret": 0.02192588895559311, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "tau": -0.07465520951292609 + }, + "1480856810/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5083154135037583, + "pairwise_causal_calibration_ece": 0.09882744363909879, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.1191024751801397, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "tau": -0.07465520951292609 + }, + "1480856810/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5110030837863524, + "pairwise_causal_calibration_ece": 0.15386022664349525, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.1493088895471595, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1480856810/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5118724897465404, + "pairwise_causal_calibration_ece": 0.05955608168203097, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0926696735019097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "tau": -0.07465520951292609 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.5128660165598996, + "pairwise_causal_calibration_ece": 0.1928660165598996, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.14698878641541877, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.52, + "pairwise_causal_calibration_confidence": 0.511941221071365, + "pairwise_causal_calibration_ece": 0.008058778928635046, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.09190610579492059, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5103616032224382, + "pairwise_causal_calibration_ece": 0.08179017465100963, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1520858341588262, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "tau": -0.07465520951292609 + }, + "1522272686/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5319219008021803, + "pairwise_causal_calibration_ece": 0.07522095634067683, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3891433321605168, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "tau": -0.07465520951292609 + }, + "1522272686/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5176703997119507, + "pairwise_causal_calibration_ece": 0.2676703997119507, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3649738510562886, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "tau": -0.07465520951292609 + }, + "1522272686/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5183770201500114, + "pairwise_causal_calibration_ece": 0.05409130586429711, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.35603713675787785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5262312992720155, + "pairwise_causal_calibration_ece": 0.3666258435851274, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.048325782527749814, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21791090071201324, + "selected_utility_gain_over_base": 0.04541558027267456, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "tau": -0.07465520951292609 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5173089481291107, + "pairwise_causal_calibration_ece": 0.26840533758517504, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06964181922005985, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.3088321089744568, + "selected_utility_gain_over_base": 0.1363367885351181, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5161816159310763, + "pairwise_causal_calibration_ece": 0.09096124121178084, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0581167593221707, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12139803171157837, + "selected_utility_gain_over_base": -0.051097288727760315, + "selector_regret": 0.09263791143894196, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "tau": -0.07465520951292609 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5123855862850035, + "pairwise_causal_calibration_ece": 0.19095701485643207, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07399457979836796, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21093030273914337, + "selected_utility_gain_over_base": -0.0033356696367263794, + "selector_regret": 0.003326788544654846, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "tau": -0.07465520951292609 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5156994405981151, + "pairwise_causal_calibration_ece": 0.23430055940188488, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06039227699080513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21256697177886963, + "selected_utility_gain_over_base": -0.001699000597000122, + "selector_regret": 0.00506339967250824, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "tau": -0.07465520951292609 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5121377255628292, + "pairwise_causal_calibration_ece": 0.012137725562829216, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09822142936424738, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018326029181480408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "tau": -0.07465520951292609 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.3331167533441346, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.38909781967835777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.46100102368298207, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1790188787/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5057916478058077, + "pairwise_causal_calibration_ece": 0.31563692362276374, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0792771393876592, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1790188787/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5062566081726894, + "pairwise_causal_calibration_ece": 0.10088624897016774, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0821139696201234, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "tau": -0.07465520951292609 + }, + "1790188787/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8636363636363636, + "pairwise_causal_calibration_confidence": 0.5082783651389248, + "pairwise_causal_calibration_ece": 0.3553579984974389, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.0426295433413653, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5154582541421188, + "pairwise_causal_calibration_ece": 0.19882746014359554, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09172847339145515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03715964034199715, + "selected_utility_gain_over_base": -0.05726582184433937, + "selector_regret": 0.0008668527007102966, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "tau": -0.07465520951292609 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5122846916174123, + "pairwise_causal_calibration_ece": 0.059143879811159095, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11469903634944659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06487039476633072, + "selected_utility_gain_over_base": -0.0295550674200058, + "selector_regret": 0.010523699223995209, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "tau": -0.07465520951292609 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5177920443784734, + "pairwise_causal_calibration_ece": 0.12506509847866953, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11123534194819423, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.057336073368787766, + "selected_utility_gain_over_base": -0.03708938881754875, + "selector_regret": 0.0014832988381385803, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "tau": -0.07465520951292609 + }, + "1845489013/train0": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5131863087960253, + "pairwise_causal_calibration_ece": 0.022527976918260362, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.056123788714222106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04260633513331413, + "selected_utility_gain_over_base": -0.06995167210698128, + "selector_regret": 0.00027813389897346497, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "tau": -0.07465520951292609 + }, + "1845489013/train1": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5137646328948755, + "pairwise_causal_calibration_ece": 0.1290925099622674, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10685267161815218, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06991276890039444, + "selected_utility_gain_over_base": -0.04264523833990097, + "selector_regret": 0.025380156934261322, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "tau": -0.07465520951292609 + }, + "1845489013/train2": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5162114675678457, + "pairwise_causal_calibration_ece": 0.30521710386072565, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11053198394490094, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0861951932311058, + "selected_utility_gain_over_base": -0.026362814009189606, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "tau": -0.07465520951292609 + }, + "1999022574/train0": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5109113095795552, + "pairwise_causal_calibration_ece": 0.16766011899187339, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11814581215768769, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "tau": -0.07465520951292609 + }, + "1999022574/train1": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.517373533290843, + "pairwise_causal_calibration_ece": 0.017373533290843013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08196834306253598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "1999022574/train2": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5201361575280914, + "pairwise_causal_calibration_ece": 0.1941495567576229, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09926036807855881, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "tau": -0.07465520951292609 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5077128468462638, + "pairwise_causal_calibration_ece": 0.20657286743945047, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15769274322565577, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "tau": -0.07465520951292609 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7777777777777778, + "pairwise_causal_calibration_confidence": 0.5098868256616201, + "pairwise_causal_calibration_ece": 0.26789095211615765, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": -0.12821380008018857, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2717533111572266, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5100908012774384, + "pairwise_causal_calibration_ece": 0.38276634157970446, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.09296986205333337, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "tau": -0.07465520951292609 + }, + "2038594191/train0": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5257244389989378, + "pairwise_causal_calibration_ece": 0.33141841814391926, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18263322741939308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15247176587581635, + "selected_utility_gain_over_base": 0.0574311688542366, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "tau": -0.07465520951292609 + }, + "2038594191/train1": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.514770372402238, + "pairwise_causal_calibration_ece": 0.264770372402238, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1571311927054344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14092111587524414, + "selected_utility_gain_over_base": 0.0458805188536644, + "selector_regret": 0.049359604716300964, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "tau": -0.07465520951292609 + }, + "2038594191/train2": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5169917573441141, + "pairwise_causal_calibration_ece": 0.12413461448697122, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1499261458574031, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10970946401357651, + "selected_utility_gain_over_base": 0.014668866991996765, + "selector_regret": 0.1390988454222679, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "tau": -0.07465520951292609 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5052995351748154, + "pairwise_causal_calibration_ece": 0.020506910444550552, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.3176949282327354, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0726270377635956, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9273729622364044, + "tau": -0.07465520951292609 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5105702026856556, + "pairwise_causal_calibration_ece": 0.09457081562854205, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.2252511010407856, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0755530372262, + "selected_candidate_index": 2.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9244469627737999, + "tau": -0.07465520951292609 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.513192964083336, + "pairwise_causal_calibration_ece": 0.06483208938690344, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.40358682075573915, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0683744624257088, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9316255375742912, + "tau": -0.07465520951292609 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5199008764839764, + "pairwise_causal_calibration_ece": 0.19438483780173788, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03336441156547287, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07517768442630768, + "selected_utility_gain_over_base": -0.011029243469238281, + "selector_regret": 0.0012469589710235596, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "tau": -0.07465520951292609 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5189283527559876, + "pairwise_causal_calibration_ece": 0.15964307581544102, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04191814555358769, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08908586204051971, + "selected_utility_gain_over_base": 0.002878934144973755, + "selector_regret": 0.0013788342475891113, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "tau": -0.07465520951292609 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5201168018122745, + "pairwise_causal_calibration_ece": 0.16297394466941734, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.033653298036953966, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07194386422634125, + "selected_utility_gain_over_base": -0.014263063669204712, + "selector_regret": 0.00708027184009552, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "tau": -0.07465520951292609 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5086109136222222, + "pairwise_causal_calibration_ece": 0.06281765780634918, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.133838155897241, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3979237377643585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "tau": -0.07465520951292609 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.507099429830539, + "pairwise_causal_calibration_ece": 0.07852800125911047, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14019095394361425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.017371907830238342, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "tau": -0.07465520951292609 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5123986202425596, + "pairwise_causal_calibration_ece": 0.13045852261458335, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0967227357905851, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.029980570077896118, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "tau": -0.07465520951292609 + }, + "278008524/train0": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5087559775718801, + "pairwise_causal_calibration_ece": 0.013479154071202815, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.07549504506362273, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0722030997276306, + "selected_candidate_index": 5.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0467599928379059, + "selected_utility_gain_over_base": -0.07824721932411194, + "selector_regret": 0.02544310688972473, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04170554131269455, + "tau": -0.07465520951292609 + }, + "278008524/train1": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5071032645290914, + "pairwise_causal_calibration_ece": 0.08573408202116861, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.007193800112369225, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1246033310890198, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1101078614592552, + "selected_utility_gain_over_base": -0.014899350702762604, + "selector_regret": 0.014495469629764557, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "278008524/train2": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5171398278381145, + "pairwise_causal_calibration_ece": 0.1514270567762001, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.01000265227293061, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.157032236456871, + "selected_candidate_index": 1.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.115786612033844, + "selected_utility_gain_over_base": -0.009220600128173828, + "selector_regret": 0.04124562442302704, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5194284844280975, + "pairwise_causal_calibration_ece": 0.016285801286188173, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.031234880094876627, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19984664022922516, + "selected_utility_gain_over_base": -0.009305715560913086, + "selector_regret": 0.0029561370611190796, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "tau": -0.07465520951292609 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5225141037494058, + "pairwise_causal_calibration_ece": 0.12034303910773714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.037469066617763, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20222051441669464, + "selected_utility_gain_over_base": -0.0069318413734436035, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "tau": -0.07465520951292609 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.518763232568162, + "pairwise_causal_calibration_ece": 0.19552248171755227, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.030969288861405195, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2035302072763443, + "selected_utility_gain_over_base": -0.005622148513793945, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "tau": -0.07465520951292609 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5062670117038048, + "pairwise_causal_calibration_ece": 0.18483844027523338, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.16113806347993082, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5113786791180399, + "pairwise_causal_calibration_ece": 0.2386213208819601, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15811820951338246, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5089676132647365, + "pairwise_causal_calibration_ece": 0.2232533275504508, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.18531182086534548, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "tau": -0.07465520951292609 + }, + "440577711/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5057012649277024, + "pairwise_causal_calibration_ece": 0.2800130207865833, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.0254871904912983, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "tau": -0.07465520951292609 + }, + "440577711/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.505418210717456, + "pairwise_causal_calibration_ece": 0.19772590302514825, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -1.067727630841523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "440577711/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.9797113666967447, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5195701592812082, + "pairwise_causal_calibration_ece": 0.1590012692902204, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.02945650465411806, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.203963503241539, + "selected_utility_gain_over_base": -0.005804136395454407, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "tau": -0.07465520951292609 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5215255698164627, + "pairwise_causal_calibration_ece": 0.2284744301835373, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05890578182706513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20609354972839355, + "selected_utility_gain_over_base": -0.0036740899085998535, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "tau": -0.07465520951292609 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5163446864153005, + "pairwise_causal_calibration_ece": 0.09079817072755658, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.017015444441914704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20165641605854034, + "selected_utility_gain_over_base": -0.008111223578453064, + "selector_regret": 0.002608165144920349, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "tau": -0.07465520951292609 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.5063024493634376, + "pairwise_causal_calibration_ece": 0.1336975506365624, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -1.155476122058963, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5086436934457946, + "pairwise_causal_calibration_ece": 0.21862903382693266, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.106462385735648, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.5071930884530261, + "pairwise_causal_calibration_ece": 0.08371600245606481, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.0541420784393922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5181208856251349, + "pairwise_causal_calibration_ece": 0.23187911437486508, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03414964989475239, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14155812561511993, + "selected_utility_gain_over_base": 0.07052158564329147, + "selector_regret": 0.022925466299057007, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "tau": -0.07465520951292609 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.512435265258071, + "pairwise_causal_calibration_ece": 0.048149550972356736, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.027362083160849926, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12263611704111099, + "selected_utility_gain_over_base": 0.05159957706928253, + "selector_regret": 0.034353144466876984, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "tau": -0.07465520951292609 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5094731430629448, + "pairwise_causal_calibration_ece": 0.2951874287772305, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.026060848101647938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13704223930835724, + "selected_utility_gain_over_base": 0.06600569933652878, + "selector_regret": 0.15684272348880768, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "tau": -0.07465520951292609 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5078873058739054, + "pairwise_causal_calibration_ece": 0.027826979840380295, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.04985292651651213, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5531623363494873, + "selected_utility_gain_over_base": 0.027802228927612305, + "selector_regret": 0.05929923057556152, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "tau": -0.07465520951292609 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5071201830818504, + "pairwise_causal_calibration_ece": 0.1499773259389932, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05236386089192557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5450301170349121, + "selected_utility_gain_over_base": 0.01967000961303711, + "selector_regret": 0.06507062911987305, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "tau": -0.07465520951292609 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5096252566953113, + "pairwise_causal_calibration_ece": 0.16894617187611727, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.012997582312052308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.4804247319698334, + "selected_utility_gain_over_base": -0.044935375452041626, + "selector_regret": 0.09008613228797913, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "tau": -0.07465520951292609 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5067509279989961, + "pairwise_causal_calibration_ece": 0.4272928465661017, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.13103319050538206, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05774742551147938, + "selected_candidate_index": 2.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03338620066642761, + "selected_utility_gain_over_base": -0.00835469365119934, + "selector_regret": 0.024361224845051765, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1078349556773901, + "tau": -0.07465520951292609 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.625, + "pairwise_causal_calibration_confidence": 0.5065516692751414, + "pairwise_causal_calibration_ece": 0.12300764930449637, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07443337753579249, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0619367565959692, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.043074944987893105, + "selected_utility_gain_over_base": 0.0013340506702661514, + "selector_regret": 0.018861811608076096, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10364562459290028, + "tau": -0.07465520951292609 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6607142857142857, + "pairwise_causal_calibration_confidence": 0.512420190557138, + "pairwise_causal_calibration_ece": 0.14829409515714775, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.05177776713392784, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08006523177027702, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04263448715209961, + "selected_utility_gain_over_base": 0.0008935928344726562, + "selector_regret": 0.037430744618177414, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08551714941859245, + "tau": -0.07465520951292609 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5055966367137085, + "pairwise_causal_calibration_ece": 0.17297479185772013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.11026859162288982, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "tau": -0.07465520951292609 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5078518602413418, + "pairwise_causal_calibration_ece": 0.11499471738419892, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14666728076214766, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "tau": -0.07465520951292609 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5081316115880623, + "pairwise_causal_calibration_ece": 0.3132969598405091, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08206888832985737, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005137942731380463, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "tau": -0.07465520951292609 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5166331169260667, + "pairwise_causal_calibration_ece": 0.23336688307393327, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0781444029185483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01332351565361023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "tau": -0.07465520951292609 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5226674723877733, + "pairwise_causal_calibration_ece": 0.09409604381634479, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.01818175818777909, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12485501170158386, + "selected_utility_gain_over_base": -0.01922038197517395, + "selector_regret": 0.025532275438308716, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "tau": -0.07465520951292609 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.525781459845288, + "pairwise_causal_calibration_ece": 0.025781459845288013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.024273861804607055, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16552868485450745, + "selected_utility_gain_over_base": 0.021453291177749634, + "selector_regret": 0.0068964362144470215, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "tau": -0.07465520951292609 + }, + "672328262/train0": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5213462849964556, + "pairwise_causal_calibration_ece": 0.19293942928925867, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1294553020456159, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.045009810477495193, + "selected_utility_gain_over_base": -0.060924429446458817, + "selector_regret": 0.014783523976802826, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "tau": -0.07465520951292609 + }, + "672328262/train1": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5156576148233183, + "pairwise_causal_calibration_ece": 0.08708618625188974, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09180646718725743, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05302006006240845, + "selected_utility_gain_over_base": -0.05291417986154556, + "selector_regret": 0.00036839768290519714, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "tau": -0.07465520951292609 + }, + "672328262/train2": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5131890723327346, + "pairwise_causal_calibration_ece": 0.022525213381551112, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11194999822009422, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0664752721786499, + "selected_utility_gain_over_base": -0.03945896774530411, + "selector_regret": 0.005690954625606537, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "tau": -0.07465520951292609 + }, + "701667486/train0": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5100395188896416, + "pairwise_causal_calibration_ece": 0.1328176239675013, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1980299443706942, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "tau": -0.07465520951292609 + }, + "701667486/train1": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5074799238651517, + "pairwise_causal_calibration_ece": 0.0074799238651517275, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1667987048169856, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "tau": -0.07465520951292609 + }, + "701667486/train2": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5089535245646244, + "pairwise_causal_calibration_ece": 0.1696179040068042, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.19998664785779915, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07465520951292609 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5050963819862415, + "pairwise_causal_calibration_ece": 0.14795352484338437, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.41293810625522775, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "tau": -0.07465520951292609 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.5172521843522638, + "pairwise_causal_calibration_ece": 0.41131924421916477, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.43548232619379196, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "tau": -0.07465520951292609 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5073556110406273, + "pairwise_causal_calibration_ece": 0.06407296038794408, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.44021912627823445, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "tau": -0.07465520951292609 + }, + "750448296/train0": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5176625382186207, + "pairwise_causal_calibration_ece": 0.13781121565484575, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.19117397742857523, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20943730572859445, + "selected_candidate_index": 2.0, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8171049058437347, + "selected_utility_gain_over_base": 0.056601578990618386, + "selector_regret": 0.006840065121650696, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1985591898361843, + "tau": -0.07465520951292609 + }, + "750448296/train1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.3452380952380952, + "pairwise_causal_calibration_confidence": 0.517466642602347, + "pairwise_causal_calibration_ece": 0.17222854736425175, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08582354011659181, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22532296180725098, + "selected_candidate_index": 3.0, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7914677709341049, + "selected_utility_gain_over_base": 0.030964444080988567, + "selector_regret": 0.03293752670288086, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1875135501225789, + "tau": -0.07465520951292609 + }, + "750448296/train2": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.515094948185793, + "pairwise_causal_calibration_ece": 0.19366637675722168, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1967269353075053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2481476068496704, + "selected_candidate_index": 1.6666666666666667, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.11175449689229329, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1598488887151082, + "tau": -0.07465520951292609 + }, + "767747424/train0": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5123633414032891, + "pairwise_causal_calibration_ece": 0.21687593622230478, + "pairwise_causal_calibration_pairs": 22.75, + "predicted_margin": -0.22610846089363168, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5789013803005219, + "selected_candidate_index": 2.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0747087560594082, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019455675035715103, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10475751012563705, + "tau": -0.07465520951292609 + }, + "767747424/train1": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.46428571428571425, + "pairwise_causal_calibration_confidence": 0.5127675241284637, + "pairwise_causal_calibration_ece": 0.09358450213861466, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.20272105535688145, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5825987188145518, + "selected_candidate_index": 1.25, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.056986487004906, + "selected_utility_gain_over_base": -0.01772226905450225, + "selector_regret": 0.022256564814597368, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1008116127923131, + "tau": -0.07465520951292609 + }, + "767747424/train2": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.4464285714285714, + "pairwise_causal_calibration_confidence": 0.5132197152821629, + "pairwise_causal_calibration_ece": 0.21218415109607414, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.18811356831313295, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5809952197596431, + "selected_candidate_index": 3.75, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0559786520898342, + "selected_utility_gain_over_base": -0.018730103969573975, + "selector_regret": 0.020647740922868252, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09709237236529589, + "tau": -0.07465520951292609 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..5a921e7f285b90603d3b9cbe0bf38ddb41ce2acd --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.0, + "base_utility": 0.4796089559793472, + "coverage": 0.8, + "execute_generated": 0.8, + "fallback_rate": 0.2, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.6304232804232804, + "pairwise_causal_calibration_confidence": 0.5135220089933253, + "pairwise_causal_calibration_ece": 0.1381376537441502, + "pairwise_causal_calibration_pairs": 27.933333333333334, + "predicted_margin": -0.02998554890632619, + "proposal_oracle_success": 0.13333333333333333, + "proposal_oracle_utility": 0.6601470321416855, + "selected_candidate_index": 4.266666666666667, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.475364543000857, + "selected_utility_gain_over_base": -0.004244412978490194, + "selector_regret": 0.18489418029785157, + "success_selector_gap": 0.13333333333333333, + "success_support_gap": 0.8666666666666667, + "support_gap": 1.3398529678583144, + "tau": -0.07465520951292606 + }, + "PickCube-v1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44047619047619047, + "pairwise_causal_calibration_confidence": 0.5226564402213808, + "pairwise_causal_calibration_ece": 0.13232755397230822, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3700514399915611, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 1.021850009759267, + "selected_candidate_index": 4.666666666666667, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 0.9781499902407328, + "tau": -0.07465520951292609 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.8032506120204925, + "coverage": 0.4533333333333333, + "execute_generated": 0.4533333333333333, + "fallback_rate": 0.5466666666666666, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.32, + "pairwise_causal_calibration_accuracy": 0.5311771561771562, + "pairwise_causal_calibration_confidence": 0.5130368946827227, + "pairwise_causal_calibration_ece": 0.15360683599994118, + "pairwise_causal_calibration_pairs": 24.16, + "predicted_margin": -0.12701778822001247, + "proposal_oracle_success": 0.17333333333333334, + "proposal_oracle_utility": 0.46170220340291657, + "selected_candidate_index": 2.6266666666666665, + "selected_success": 0.36, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7987058012435834, + "selected_utility_gain_over_base": -0.004544810776909192, + "selector_regret": 0.02149127927919229, + "success_selector_gap": 0.0, + "success_support_gap": 0.3466666666666667, + "support_gap": 0.6881442584097386, + "tau": -0.07465520951292605 + }, + "PushCube-v1": { + "base_success": 0.8333333333333334, + "base_utility": 1.6734557673335075, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.16666666666666666, + "pairwise_causal_calibration_accuracy": 0.5866721078485784, + "pairwise_causal_calibration_confidence": 0.5094999341420311, + "pairwise_causal_calibration_ece": 0.1509193203080439, + "pairwise_causal_calibration_pairs": 23.833333333333332, + "predicted_margin": -1.0899306043385468, + "proposal_oracle_success": 0.5555555555555556, + "proposal_oracle_utility": 1.1790093589160178, + "selected_candidate_index": 2.5555555555555554, + "selected_success": 0.8333333333333334, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.6734557673335075, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32654423266649246, + "success_selector_gap": 0.16666666666666666, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6220440144340197, + "tau": -0.07465520951292606 + }, + "StackCube-v1": { + "base_success": 0.09090909090909091, + "base_utility": 0.5003613992170854, + "coverage": 0.45454545454545453, + "execute_generated": 0.45454545454545453, + "fallback_rate": 0.5454545454545454, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6060606060606061, + "pairwise_causal_calibration_accuracy": 0.5222077922077923, + "pairwise_causal_calibration_confidence": 0.5154809843682674, + "pairwise_causal_calibration_ece": 0.14110861032541971, + "pairwise_causal_calibration_pairs": 27.818181818181817, + "predicted_margin": -0.09559995857533575, + "proposal_oracle_success": 0.18181818181818182, + "proposal_oracle_utility": 0.634040683056369, + "selected_candidate_index": 3.0, + "selected_success": 0.09090909090909091, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.4970153953992959, + "selected_utility_gain_over_base": -0.003346003817789482, + "selector_regret": 0.18124081239555823, + "success_selector_gap": 0.12121212121212122, + "success_support_gap": 0.8181818181818182, + "support_gap": 1.365959316943631, + "tau": -0.07465520951292605 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..e4103630d0b557f4d19368e46f189f376d90de3a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.4236 & 0.5764 & 0.3333 & 0.3333 & 0.2222 & 0.3611 & 0.5139 & 0.0625 & 0.0344 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..41994b5633aeeb803760e5ffc1d9c8d7e9b27436 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_perdim_trainmax_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=0.0 +tau=-0.074655 +calibration_input=runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..940cc12de9dd07aecc255a9adae1b1066d7855be --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test --feature-set context --target success --k 8 --bootstrap-samples 1000 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d49af63cb8676283dc39f331ada8763fe155b96e --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: context +fit_objective: pointwise +k: 8 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: None +success_bonus: 1.0 +target: success +threshold_scope: global diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d91fbacd145085f2d9a143f07cf1791ff492f9fc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "130d86b8c5ba5dc901ee48835454693bf42c361803c01f6b2d80590cb2493491", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "88d4bfe8c95f8d1579cbe59096cf665dc5f2dfdd99481a386a25e1f3cfa2b09d", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..58e60527653c049a1edc75cf9002db1286feb6ae --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3df280f42de841deb69e1683623ef753fdd47773 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics.json @@ -0,0 +1,8987 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.013888888888888888, + "fallback_rate": 0.9861111111111112, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3020833333333333, + "selected_success_gain_over_base": 0.003472222222222222, + "selected_utility": 0.7552938958785186, + "selector_regret": 0.07355554220784041, + "success_selector_gap": 0.041666666666666664, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_summary": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.013888888888888888, + "fallback_rate": 0.9861111111111112, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "pairwise_causal_calibration_accuracy": 0.49986493787142083, + "pairwise_causal_calibration_confidence": 0.5088942931416934, + "pairwise_causal_calibration_ece": 0.009029355270272599, + "pairwise_causal_calibration_pairs": 7404.0, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3020833333333333, + "selected_success_gain_over_base": 0.003472222222222222, + "selected_utility": 0.7552938958785186, + "selector_regret": 0.07355554220784041, + "success_selector_gap": 0.041666666666666664, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.3333333333333333, + "base_utility": 0.8338337515791258, + "coverage": 0.006944444444444444, + "fallback_rate": 0.9930555555555556, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.3611111111111111, + "pairwise_causal_calibration_accuracy": 0.4781540141998908, + "pairwise_causal_calibration_confidence": 0.5096421638170766, + "pairwise_causal_calibration_ece": 0.03148814961718582, + "pairwise_causal_calibration_pairs": 3662.0, + "proposal_oracle_success": 0.2222222222222222, + "proposal_oracle_utility": 0.6232009150553495, + "selected_success": 0.3263888888888889, + "selected_success_gain_over_base": -0.006944444444444444, + "selected_utility": 0.8208790636724896, + "selector_regret": 0.11845623132669264, + "success_selector_gap": 0.0625, + "success_support_gap": 0.5138888888888888, + "support_gap": 0.9091424554741631 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.182546837171281, + 1.8836221616495297, + -1.0601087908747155e-18, + 3.5, + 0.125, + 1.2092013888888888, + 0.7583173435164257, + 1.9059527259604592, + 8.0, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.36258169934639883, + -0.4588235294117674, + 0.1346405228758154, + 0.3127450980392166, + -0.5231481481481283, + -0.5494008714596817, + 0.5098039215686088, + -0.08622004357298363, + 1.0, + 0.2707248263888889, + 0.2306857638888889 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "target_task_hash_00", + "target_task_hash_01", + "target_task_hash_02", + "target_task_hash_03", + "target_task_hash_04", + "target_task_hash_05", + "target_task_hash_06", + "target_task_hash_07", + "source_task_hash_00", + "source_task_hash_01", + "source_task_hash_02", + "source_task_hash_03", + "source_task_hash_04", + "source_task_hash_05", + "source_task_hash_06", + "source_task_hash_07", + "instruction_hash_00", + "instruction_hash_01", + "instruction_hash_02", + "instruction_hash_03", + "instruction_hash_04", + "instruction_hash_05", + "instruction_hash_06", + "instruction_hash_07", + "source_target_same_task", + "instruction_chars_per_128", + "instruction_words_per_32" + ], + "feature_set": "context", + "feature_std": [ + 1.0, + 2.4903288136152946, + 2.1478743840800227, + 0.999998700632758, + 2.29128884747792, + 0.3307199138830738, + 1.490882562848811, + 0.3689689239790769, + 0.7063143432867478, + 1e-06, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37487873363847424, + 0.3443939715604908, + 0.32513795823134695, + 0.3102098787644046, + 0.4154730332740044, + 0.3938110555196154, + 0.4628208676716171, + 0.7177468630079228, + 1e-06, + 0.03103759015348271, + 0.02828865844409628 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 2304, + "num_fit_rows": 2304, + "num_pairwise_rows": 0, + "num_pointwise_rows": 2304, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 8, + "num_calibration_candidates": 2304, + "num_calibration_rows": 288, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.49986493787142083, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.009029355270272599, + "accuracy": 0.49986493787142083, + "confidence": 0.5088942931416934, + "count": 7404, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.009029355270272599, + "mean_confidence": 0.5088942931416934, + "n_bins": 10, + "num_pairs": 7404, + "num_rows": 288 + }, + "eval": { + "accuracy": 0.4781540141998908, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.03148814961718582, + "accuracy": 0.4781540141998908, + "confidence": 0.5096421638170766, + "count": 3662, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.03148814961718582, + "mean_confidence": 0.5096421638170766, + "n_bins": 10, + "num_pairs": 3662, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5025021287537501, + "pairwise_causal_calibration_ece": 0.07393070018232156, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12745811668995927, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "seed": "2012211287/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5097271036094443, + "pairwise_causal_calibration_ece": 0.11686996075230144, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17403822005730368, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "seed": "307068879/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5050884488962025, + "pairwise_causal_calibration_ece": 0.07651702032477398, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12685629258227518, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "seed": "1073161778/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5065245373000641, + "pairwise_causal_calibration_ece": 0.006524537300064082, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09283658594391792, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "seed": "608016290/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08710145950317383, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5109682345626951, + "pairwise_causal_calibration_ece": 0.18953966313412368, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16069187570998772, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "seed": "441046872/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5320529802017496, + "pairwise_causal_calibration_ece": 0.353481551630321, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.49245171577741104, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "seed": "1522272686/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13452494144439697, + "selected_utility_gain_over_base": -1.865475058555603, + "selector_regret": 0.38879501819610596, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "task_id": "PickCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.2736338609992832, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5101735478776616, + "pairwise_causal_calibration_ece": 0.3673164050205187, + "pairwise_causal_calibration_pairs": 7.0, + "predicted_margin": 0.04882028591256058, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5088280313121364, + "pairwise_causal_calibration_ece": 0.044542317026422085, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11446801340756732, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "seed": "1461069713/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5104578702131527, + "pairwise_causal_calibration_ece": 0.08188644164172415, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05295439005629449, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.044337254017591476, + "seed": "626486813/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01164356991648674, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2090834565460682, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5050441593186851, + "pairwise_causal_calibration_ece": 0.04075844503297077, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18828274270152773, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14525407552719116, + "seed": "2054899126/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8547459244728088, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5094622393113466, + "pairwise_causal_calibration_ece": 0.009462239311346599, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.008566982900256922, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "seed": "672328262/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5127606448590804, + "pairwise_causal_calibration_ece": 0.16581078371234825, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.27583127727281853, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2748139500617981, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07782270014286041, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5092617090990478, + "pairwise_causal_calibration_ece": 0.009261709099047777, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06183430692022682, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14440619945526123, + "seed": "278008524/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0834110826253891, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.507729136499722, + "pairwise_causal_calibration_ece": 0.007729136499721956, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07034244814348488, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "seed": "1845489013/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5108261099462067, + "pairwise_causal_calibration_ece": 0.08225468137477815, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04952121164092821, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07115759700536728, + "seed": "626486813/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05078810453414917, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02036949247121811, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.006586454808712006, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5139135269880074, + "pairwise_causal_calibration_ece": 0.08534209841657886, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.33222121175049485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.011163711547851562, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2515503764152527, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5089117415058044, + "pairwise_causal_calibration_ece": 0.008911741505804427, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.024574618327409238, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "seed": "585904130/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09344705194234848, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5087967782363918, + "pairwise_causal_calibration_ece": 0.1697746503350368, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15640249398420367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "seed": "141984615/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5120356454165671, + "pairwise_causal_calibration_ece": 0.11917850255942425, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10275117316662587, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25842753052711487, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.19032493233680725, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7415724694728851, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5126032019004112, + "pairwise_causal_calibration_ece": 0.1302539409567317, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.005906212010217932, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "seed": "636680137/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5042221136632125, + "pairwise_causal_calibration_ece": 0.1470792565203553, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05698912113408571, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.029627859592437744, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.16747966408729553, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07692307692307693, + "pairwise_causal_calibration_confidence": 0.5080201327824784, + "pairwise_causal_calibration_ece": 0.43109705585940145, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.016525300307342672, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5141674398337075, + "pairwise_causal_calibration_ece": 0.19273886840513604, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04432455250231335, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "seed": "2038594191/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0574311688542366, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.514570981225559, + "pairwise_causal_calibration_ece": 0.1640004473458696, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2086674653720124, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13244229555130005, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09154719114303589, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.2813234303126597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5094045097956438, + "pairwise_causal_calibration_ece": 0.02630977591864192, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19002955609852606, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.0666710197931548, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5144342311662078, + "pairwise_causal_calibration_ece": 0.2355657688337922, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20313397288962703, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04541558027267456, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5076183712392905, + "pairwise_causal_calibration_ece": 0.0076183712392905445, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.022563707465889905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "seed": "1822210471/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5072745787005666, + "pairwise_causal_calibration_ece": 0.04298886441485228, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04533265312764892, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23744209110736847, + "seed": "750448296/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7625579088926315, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.505089963282911, + "pairwise_causal_calibration_ece": 0.2806243224313747, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.28011676269294883, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "seed": "440577711/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5048732080022066, + "pairwise_causal_calibration_ece": 0.14773035085934944, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21794545086954079, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "seed": "1324642411/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5036856098684014, + "pairwise_causal_calibration_ece": 0.21060010441731292, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20654914374778957, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.506033274055822, + "pairwise_causal_calibration_ece": 0.2796810116584637, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20374180832437194, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "seed": "1261930456/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44, + "pairwise_causal_calibration_confidence": 0.5079336789116323, + "pairwise_causal_calibration_ece": 0.06793367891163232, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.265503929801179, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5065305341473837, + "pairwise_causal_calibration_ece": 0.006530534147383693, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2674046465805726, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "seed": "1480856810/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5121903376361622, + "pairwise_causal_calibration_ece": 0.2621903376361622, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.026008561069899745, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "seed": "2061635888/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5096348011231283, + "pairwise_causal_calibration_ece": 0.15249194398027116, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0803760660625997, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "seed": "655884605/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01332351565361023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5130393809952023, + "pairwise_causal_calibration_ece": 0.08446795242377375, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1065139428446398, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "seed": "1113371261/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5133443564097337, + "pairwise_causal_calibration_ece": 0.16522707216169485, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11674964401723253, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "seed": "1578990937/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.76, + "pairwise_causal_calibration_confidence": 0.5045165731476825, + "pairwise_causal_calibration_ece": 0.2554834268523175, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.09788706834443102, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5075662677800439, + "pairwise_causal_calibration_ece": 0.02814801793424182, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14765513870898905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "seed": "211245842/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3979237377643585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.511440126674405, + "pairwise_causal_calibration_ece": 0.2742741590398807, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11740056999820042, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "seed": "1151410346/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3400209844112396, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5161397587413304, + "pairwise_causal_calibration_ece": 0.1232826158841876, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1385357960706925, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "seed": "1324133122/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5064903449632235, + "pairwise_causal_calibration_ece": 0.02922394075106216, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08941960203365407, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "seed": "701667486/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5074318377589786, + "pairwise_causal_calibration_ece": 0.1354253050981643, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12581173470043255, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5113663967558075, + "pairwise_causal_calibration_ece": 0.18993782532723608, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0923468998673774, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "seed": "1999022574/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5023600590177417, + "pairwise_causal_calibration_ece": 0.0579156145732973, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.1339440868297946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2717533111572266, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.506525070880443, + "pairwise_causal_calibration_ece": 0.04223935659472866, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15887380976711568, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "seed": "307068879/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.506364326571218, + "pairwise_causal_calibration_ece": 0.0650642448573534, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1717951848740798, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "seed": "1073161778/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5094605835953956, + "pairwise_causal_calibration_ece": 0.2594605835953956, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16705607246529852, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "seed": "608016290/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08474063873291016, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5075221409445348, + "pairwise_causal_calibration_ece": 0.1503792838016777, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13932622413588797, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "seed": "441046872/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5150111595103795, + "pairwise_causal_calibration_ece": 0.050725445224665155, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3716076139049983, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "seed": "1522272686/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "task_id": "PickCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.37283460103555477, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5073871696939516, + "pairwise_causal_calibration_ece": 0.02832711602033411, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11394919615654472, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5019504846479711, + "pairwise_causal_calibration_ece": 0.2519504846479711, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09666305797857698, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "seed": "1461069713/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5100063475789608, + "pairwise_causal_calibration_ece": 0.2242920618646751, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11227206678401797, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06662949174642563, + "seed": "626486813/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03393580764532089, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18679121881723404, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5093597426064234, + "pairwise_causal_calibration_ece": 0.22364545689213766, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.27090137381780155, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1511060744524002, + "seed": "2054899126/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8488939255475998, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5192789991993754, + "pairwise_causal_calibration_ece": 0.05214957222919603, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16735307792538862, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "seed": "672328262/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5113406771651661, + "pairwise_causal_calibration_ece": 0.13151646569197684, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3440861057654468, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2738197147846222, + "seed": "767747424/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07682846486568451, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5137154678536687, + "pairwise_causal_calibration_ece": 0.16485596071775988, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03466917031062028, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24920666217803955, + "seed": "278008524/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5041162134354321, + "pairwise_causal_calibration_ece": 0.0041162134354321145, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03038327939178103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "seed": "1845489013/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5075617708399499, + "pairwise_causal_calibration_ece": 0.028152514874335788, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04381683733214927, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05724402144551277, + "seed": "626486813/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05078810453414917, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006455916911363602, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.020500030368566513, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5071731281092784, + "pairwise_causal_calibration_ece": 0.15003027096642124, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13770727436714467, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.013422667980194092, + "seed": "767747424/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24929141998291016, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082300568513344, + "pairwise_causal_calibration_ece": 0.07965862827990583, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04771232196356422, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "seed": "585904130/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08595272153615952, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5148179816508425, + "pairwise_causal_calibration_ece": 0.09232487549201462, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16955591458280203, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5138571660281585, + "pairwise_causal_calibration_ece": 0.12899997682898445, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.178434027240592, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2347063571214676, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.16660375893115997, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7652936428785324, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5068342927158962, + "pairwise_causal_calibration_ece": 0.11397714985875335, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.01736105409945201, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "seed": "636680137/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5081697036237933, + "pairwise_causal_calibration_ece": 0.3295982750523647, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11449832880221902, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04315249249339104, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15395503118634224, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5124704817994062, + "pairwise_causal_calibration_ece": 0.04818476751369194, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19880639201897846, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "seed": "1375492786/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5220880545588878, + "pairwise_causal_calibration_ece": 0.04934051686968355, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11815756078684783, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "seed": "2038594191/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09524012356996536, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5157832357059579, + "pairwise_causal_calibration_ece": 0.0872118071345293, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22465033902852502, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23850953578948975, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.025102153420448303, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.26505372853914666, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5141512545538647, + "pairwise_causal_calibration_ece": 0.3072773168747067, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2604839026968317, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.06973587348782971, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5137996065010031, + "pairwise_causal_calibration_ece": 0.1290575363561398, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3351387925434473, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1363367885351181, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5092194643264373, + "pairwise_causal_calibration_ece": 0.009219464326437277, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.016662843715525302, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "seed": "1822210471/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5134875428215862, + "pairwise_causal_calibration_ece": 0.022226742892699547, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15754118060079794, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2027529925107956, + "seed": "750448296/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7972470074892044, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.15384615384615385, + "pairwise_causal_calibration_confidence": 0.5069423941005992, + "pairwise_causal_calibration_ece": 0.3530962402544453, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.39808073658745674, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.507157276642487, + "pairwise_causal_calibration_ece": 0.2849350544202648, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.347647788530772, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.506979467301554, + "pairwise_causal_calibration_ece": 0.13587767555558894, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22799459496707442, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "seed": "1790188787/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5043609091246408, + "pairwise_causal_calibration_ece": 0.04007519483892652, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2409286430045787, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "seed": "1261930456/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.18181818181818182, + "pairwise_causal_calibration_confidence": 0.5053331100145789, + "pairwise_causal_calibration_ece": 0.3235149281963971, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.2742130213071048, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5135431948467644, + "pairwise_causal_calibration_ece": 0.12068605198962151, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3753954390883053, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5130608055518229, + "pairwise_causal_calibration_ece": 0.19163223412325142, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1199851428584793, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "seed": "2061635888/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.004257768392562866, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5078234193563246, + "pairwise_causal_calibration_ece": 0.04353770507061028, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07436604773890146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "seed": "655884605/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006311893463134766, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5096998710858021, + "pairwise_causal_calibration_ece": 0.13315727177134085, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07977563885669328, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "seed": "1113371261/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5081272058598021, + "pairwise_causal_calibration_ece": 0.008127205859802089, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07562416744203525, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "seed": "1578990937/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0033643990755081177, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5059429296866168, + "pairwise_causal_calibration_ece": 0.10594292968661678, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.14589984808245227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5069815835180018, + "pairwise_causal_calibration_ece": 0.06444698791056958, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11500634632910782, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "seed": "211245842/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.017371907830238342, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5034160561308088, + "pairwise_causal_calibration_ece": 0.21086965815490555, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08293417068727207, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "seed": "1151410346/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.015453755855560303, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5152712504056621, + "pairwise_causal_calibration_ece": 0.19384267897709068, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1825637783093442, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "seed": "1324133122/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.00014631450176239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5093624359297296, + "pairwise_causal_calibration_ece": 0.08079100735830108, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1566669781571995, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "seed": "701667486/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5071759937249536, + "pairwise_causal_calibration_ece": 0.11431885086781074, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11502862125762614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5079505274739976, + "pairwise_causal_calibration_ece": 0.09919232966885949, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09800642070089008, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5075402669960806, + "pairwise_causal_calibration_ece": 0.20674544728963373, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13101225256780796, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "seed": "2012211287/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5115356866600222, + "pairwise_causal_calibration_ece": 0.29724997237430795, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18354981650868932, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "seed": "307068879/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5038865127934079, + "pairwise_causal_calibration_ece": 0.0038865127934079036, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06826458778698574, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274482011795044, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5060318822435502, + "pairwise_causal_calibration_ece": 0.13682526061359268, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04324215665774371, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "seed": "608016290/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0451507568359375, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5114229219309399, + "pairwise_causal_calibration_ece": 0.22570863621665416, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15405027288626078, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "seed": "441046872/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "task_id": "LiftPegUpright-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5129514889290346, + "pairwise_causal_calibration_ece": 0.08438006035760609, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3918486005289839, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1522272686/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PickCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.3199392510777099, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5069897005433915, + "pairwise_causal_calibration_ece": 0.078418271971963, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08029034513694427, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.511070602144867, + "pairwise_causal_calibration_ece": 0.011070602144866992, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25927594010053956, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "seed": "1461069713/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5152217086371489, + "pairwise_causal_calibration_ece": 0.0919211485057082, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10150031510484146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10755517333745956, + "seed": "626486813/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07486148923635483, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1458655372262001, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5093497250220118, + "pairwise_causal_calibration_ece": 0.09779313212084528, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17792222804262553, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13674892485141754, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8632510751485825, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5164354775378074, + "pairwise_causal_calibration_ece": 0.12357833468066454, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0007002341456026143, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "seed": "672328262/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5096610381436336, + "pairwise_causal_calibration_ece": 0.06176753328493778, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.24967274951056762, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25252875685691833, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05553750693798065, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5125993969646946, + "pairwise_causal_calibration_ece": 0.2731148887495911, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12677738319378704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.31406447291374207, + "seed": "278008524/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06405004858970642, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5045771512016884, + "pairwise_causal_calibration_ece": 0.21886286548740275, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.022266377420460608, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "seed": "1845489013/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.514508346199901, + "pairwise_causal_calibration_ece": 0.021205939514384653, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.043302703012402396, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05257529020309448, + "seed": "626486813/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05078810453414917, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0017871856689453125, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.025168761610984802, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5105787685566773, + "pairwise_causal_calibration_ece": 0.13227837430046563, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15948663399221513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.01747530698776245, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2452387809753418, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5086682417290903, + "pairwise_causal_calibration_ece": 0.36581109887194746, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0138091278166185, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "seed": "585904130/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.22284842282533646, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5149845754360756, + "pairwise_causal_calibration_ece": 0.4078417182932184, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15743680246179145, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "seed": "141984615/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5157960380229301, + "pairwise_causal_calibration_ece": 0.05563253340564134, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12164452195336312, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.4033660888671875, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3352634906768799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.5966339111328125, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5084592529747467, + "pairwise_causal_calibration_ece": 0.0629693184538247, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04118746392624513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "seed": "636680137/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005137942731380463, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5059219972070264, + "pairwise_causal_calibration_ece": 0.20836371707868795, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.02784296613374379, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053976815193891525, + "seed": "767747424/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14313070848584175, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5044970161528288, + "pairwise_causal_calibration_ece": 0.07592558758140028, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08992311034993285, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "seed": "1375492786/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5174229574741631, + "pairwise_causal_calibration_ece": 0.23170867175987742, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.030673233140460502, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.15376771241426468, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5095762469450044, + "pairwise_causal_calibration_ece": 0.08100481837357582, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.24891530091435654, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17124910652637482, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05274038016796112, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.23555067247628625, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5072221251259937, + "pairwise_causal_calibration_ece": 0.1500792679831366, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1954184269801334, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "seed": "747260437/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.16303570045825555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5129772970585401, + "pairwise_causal_calibration_ece": 0.012977297058540116, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21570629252906995, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "seed": "1549324805/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04154062271118164, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5053265663635883, + "pairwise_causal_calibration_ece": 0.041040852077874, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.02439268371011942, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "seed": "1822210471/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5201227230385157, + "pairwise_causal_calibration_ece": 0.1227344198186272, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2778965514859033, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1698276251554489, + "seed": "750448296/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.830172374844551, + "task_id": "PullCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.30762508813304046, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5096750531281539, + "pairwise_causal_calibration_ece": 0.009675053128153865, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": 0.14571765838194953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8181818181818182, + "pairwise_causal_calibration_confidence": 0.5077554541172227, + "pairwise_causal_calibration_ece": 0.31042636406459556, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.2395575582689906, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5048228430200322, + "pairwise_causal_calibration_ece": 0.10232001412282488, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1368899848611159, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "seed": "1261930456/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2727272727272727, + "pairwise_causal_calibration_confidence": 0.5049476605435552, + "pairwise_causal_calibration_ece": 0.23222038781628251, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.20165171424330564, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5049749254678583, + "pairwise_causal_calibration_ece": 0.14783206832500112, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.23055093221445233, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "seed": "1480856810/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "task_id": "PushCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5104133544298581, + "pairwise_causal_calibration_ece": 0.010413354429858135, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11936751939751099, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "seed": "2061635888/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5075199240973761, + "pairwise_causal_calibration_ece": 0.2932342098116618, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09985821555853253, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "seed": "655884605/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.028349727392196655, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5110306378253284, + "pairwise_causal_calibration_ece": 0.15388778068247128, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0920288600650524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "seed": "1113371261/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5092591404892353, + "pairwise_causal_calibration_ece": 0.04497342620352096, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1280718516828302, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "seed": "1578990937/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018326029181480408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5075650128039051, + "pairwise_causal_calibration_ece": 0.1352921300532378, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21186473269066222, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "seed": "1509980664/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5062349767915747, + "pairwise_causal_calibration_ece": 0.1009078803512824, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17147291875581502, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "seed": "211245842/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.029980570077896118, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5050707909765347, + "pairwise_causal_calibration_ece": 0.13778635188060817, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1106374656498088, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "seed": "1151410346/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3345368802547455, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.510107847781337, + "pairwise_causal_calibration_ece": 0.3315364192099084, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12394704754390919, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "seed": "1324133122/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013924404978752136, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5090635477222879, + "pairwise_causal_calibration_ece": 0.0980793094205692, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2544048252784154, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5068073932444507, + "pairwise_causal_calibration_ece": 0.18537882181587922, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15894963977787818, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "seed": "412102563/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5098924941980084, + "pairwise_causal_calibration_ece": 0.3313210656265798, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16067966181951557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "seed": "1999022574/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "task_id": "StackCube-v1", + "tau": 0.41382592280226843, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 0.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3958333333333333, + "low": 0.2277777777777778, + "mean": 0.3083333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3083333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8053333333333332, + "low": 0.10836363636363637, + "mean": 0.45684848484848484, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.45684848484848484, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.4097222222222222, + "low": 0.2569444444444444, + "mean": 0.3333333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9566364926286041, + "low": 0.6609309806488455, + "mean": 0.8028034761858482, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.8028034761858482, + "per_seed": { + "1073161778/train0": 0.725517988204956, + "1073161778/train1": 0.725517988204956, + "1073161778/train2": 0.725517988204956, + "1113371261/train0": 0.11109928786754608, + "1113371261/train1": 0.11109928786754608, + "1113371261/train2": 0.11109928786754608, + "1151410346/train0": 0.2284735143184662, + "1151410346/train1": 0.2284735143184662, + "1151410346/train2": 0.2284735143184662, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.084227055311203, + "1324133122/train2": 0.084227055311203, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.16783273220062256, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.22852759063243866, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.17249532043933868, + "1549324805/train1": 0.17249532043933868, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21426597237586975, + "1578990937/train1": 0.21426597237586975, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.09442546218633652, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.09442546218633652, + "1845489013/train0": 0.11255800724029541, + "1845489013/train1": 0.11255800724029541, + "1845489013/train2": 0.11255800724029541, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.09504059702157974, + "2038594191/train1": 0.09504059702157974, + "2038594191/train2": 0.09504059702157974, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.08620692789554596, + "2061635888/train1": 0.08620692789554596, + "2061635888/train2": 0.08620692789554596, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.1250072121620178, + "278008524/train1": 1.1250072121620178, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.20915235579013824, + "307068879/train1": 0.20915235579013824, + "307068879/train2": 0.20915235579013824, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.2097676396369934, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2097676396369934, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.07103653997182846, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.07103653997182846, + "608016290/train0": 0.525360107421875, + "608016290/train1": 0.525360107421875, + "608016290/train2": 0.525360107421875, + "626486813/train0": 0.04174089431762695, + "626486813/train1": 0.04174089431762695, + "626486813/train2": 0.04174089431762695, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.1440753936767578, + "655884605/train2": 0.1440753936767578, + "672328262/train0": 0.10593423992395401, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.10593423992395401, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7605033268531164, + "750448296/train1": 0.7605033268531164, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0747087560594082, + "767747424/train1": 1.0747087560594082, + "767747424/train2": 1.0747087560594082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6347634333101186, + "low": 0.5526382644826715, + "mean": 1.0913353469100866, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.0913353469100866, + "per_task": { + "LiftPegUpright-v1": 0.4796089559793472, + "PickCube-v1": 2.0, + "PullCube-v1": 0.8032506120204925, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.5003613992170854 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9699174793851044, + "low": 0.6952519603073597, + "mean": 0.8338337515791258, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.025, + "low": 0.0, + "mean": 0.008333333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.008333333333333333, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2, + "low": 0.0, + "mean": 0.06666666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.06666666666666667, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.3333333333333333, + "PullCube-v1": 0.0, + "PushCube-v1": 0.0, + "StackCube-v1": 0.0 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.020833333333333332, + "low": 0.0, + "mean": 0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.025, + "low": 0.0, + "mean": 0.008333333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.008333333333333333, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2, + "low": 0.0, + "mean": 0.06666666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.06666666666666667, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.3333333333333333, + "PullCube-v1": 0.0, + "PushCube-v1": 0.0, + "StackCube-v1": 0.0 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.020833333333333332, + "low": 0.0, + "mean": 0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.975, + "mean": 0.9916666666666667, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.9916666666666667, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 1.0, + "750448296/train1": 1.0, + "750448296/train2": 1.0, + "767747424/train0": 1.0, + "767747424/train1": 1.0, + "767747424/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.7999999999999999, + "mean": 0.9333333333333332, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9333333333333332, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 0.6666666666666666, + "PullCube-v1": 1.0, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 1.0, + "low": 0.9791666666666666, + "mean": 0.9930555555555556, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.45138888888888895, + "low": 0.2833333333333333, + "mean": 0.3631944444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3631944444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 1.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4609696969696969, + "low": 0.1, + "mean": 0.2852121212121212, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2852121212121212, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.6060606060606061 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.4444444444444444, + "low": 0.2847222222222222, + "mean": 0.3611111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5049892755711722, + "low": 0.4403314310210863, + "mean": 0.4702836315982868, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.4702836315982868, + "per_seed": { + "1073161778/train0": 0.42857142857142855, + "1073161778/train1": 0.5714285714285714, + "1073161778/train2": 0.5, + "1113371261/train0": 0.42857142857142855, + "1113371261/train1": 0.6428571428571429, + "1113371261/train2": 0.35714285714285715, + "1151410346/train0": 0.7857142857142857, + "1151410346/train1": 0.7142857142857143, + "1151410346/train2": 0.6428571428571429, + "1261930456/train0": 0.7857142857142857, + "1261930456/train1": 0.4642857142857143, + "1261930456/train2": 0.6071428571428571, + "1324133122/train0": 0.39285714285714285, + "1324133122/train1": 0.32142857142857145, + "1324133122/train2": 0.17857142857142858, + "1324642411/train0": 0.35714285714285715, + "1324642411/train1": 0.2222222222222222, + "1324642411/train2": 0.5, + "1375492786/train0": 0.07692307692307693, + "1375492786/train1": 0.4642857142857143, + "1375492786/train2": 0.42857142857142855, + "141984615/train0": 0.6785714285714286, + "141984615/train1": 0.6071428571428571, + "141984615/train2": 0.10714285714285714, + "1461069713/train0": 0.4642857142857143, + "1461069713/train1": 0.25, + "1461069713/train2": 0.5, + "1480856810/train0": 0.5, + "1480856810/train1": 0.39285714285714285, + "1480856810/train2": 0.35714285714285715, + "1509980664/train0": 0.76, + "1509980664/train1": 0.4, + "1509980664/train2": 0.6428571428571429, + "1522272686/train0": 0.17857142857142858, + "1522272686/train1": 0.4642857142857143, + "1522272686/train2": 0.42857142857142855, + "1549324805/train0": 0.75, + "1549324805/train1": 0.6428571428571429, + "1549324805/train2": 0.5, + "1578990937/train0": 0.6785714285714286, + "1578990937/train1": 0.5, + "1578990937/train2": 0.4642857142857143, + "1790188787/train0": 0.7142857142857143, + "1790188787/train1": 0.6428571428571429, + "1790188787/train2": 0.8181818181818182, + "1822210471/train0": 0.5, + "1822210471/train1": 0.5, + "1822210471/train2": 0.4642857142857143, + "1845489013/train0": 0.5, + "1845489013/train1": 0.5, + "1845489013/train2": 0.2857142857142857, + "1999022574/train0": 0.32142857142857145, + "1999022574/train1": 0.6071428571428571, + "1999022574/train2": 0.17857142857142858, + "2012211287/train0": 0.42857142857142855, + "2012211287/train1": 0.4444444444444444, + "2012211287/train2": 0.7142857142857143, + "2038594191/train0": 0.32142857142857145, + "2038594191/train1": 0.5714285714285714, + "2038594191/train2": 0.2857142857142857, + "2054899126/train0": 0.4642857142857143, + "2054899126/train1": 0.2857142857142857, + "2054899126/train2": 0.6071428571428571, + "2061635888/train0": 0.25, + "2061635888/train1": 0.32142857142857145, + "2061635888/train2": 0.5, + "211245842/train0": 0.5357142857142857, + "211245842/train1": 0.5714285714285714, + "211245842/train2": 0.6071428571428571, + "278008524/train0": 0.5, + "278008524/train1": 0.6785714285714286, + "278008524/train2": 0.7857142857142857, + "307068879/train0": 0.39285714285714285, + "307068879/train1": 0.4642857142857143, + "307068879/train2": 0.21428571428571427, + "412102563/train0": 0.6428571428571429, + "412102563/train1": 0.39285714285714285, + "412102563/train2": 0.32142857142857145, + "440577711/train0": 0.7857142857142857, + "440577711/train1": 0.15384615384615385, + "441046872/train0": 0.32142857142857145, + "441046872/train1": 0.35714285714285715, + "441046872/train2": 0.2857142857142857, + "509430392/train0": 0.44, + "509430392/train1": 0.18181818181818182, + "509430392/train2": 0.2727272727272727, + "585904130/train0": 0.5, + "585904130/train1": 0.42857142857142855, + "585904130/train2": 0.14285714285714285, + "608016290/train0": 0.5, + "608016290/train1": 0.25, + "608016290/train2": 0.6428571428571429, + "626486813/train0": 0.42857142857142855, + "626486813/train1": 0.4107142857142857, + "626486813/train2": 0.5714285714285714, + "636680137/train0": 0.6428571428571429, + "636680137/train1": 0.39285714285714285, + "636680137/train2": 0.5714285714285714, + "655884605/train0": 0.35714285714285715, + "655884605/train1": 0.4642857142857143, + "655884605/train2": 0.21428571428571427, + "672328262/train0": 0.5, + "672328262/train1": 0.5714285714285714, + "672328262/train2": 0.39285714285714285, + "701667486/train0": 0.5357142857142857, + "701667486/train1": 0.42857142857142855, + "701667486/train2": 0.6071428571428571, + "747260437/train0": 0.5357142857142857, + "747260437/train1": 0.8214285714285714, + "747260437/train2": 0.35714285714285715, + "750448296/train0": 0.5119047619047619, + "750448296/train1": 0.5357142857142857, + "750448296/train2": 0.5476190476190476, + "767747424/train0": 0.4017857142857143, + "767747424/train1": 0.4285714285714286, + "767747424/train2": 0.5892857142857143 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.48151094656977006, + "low": 0.3975868271162389, + "mean": 0.44684157237098415, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.44684157237098415, + "per_task": { + "LiftPegUpright-v1": 0.4343915343915344, + "PickCube-v1": 0.35714285714285715, + "PullCube-v1": 0.48276723276723277, + "PushCube-v1": 0.4821140297610885, + "StackCube-v1": 0.47779220779220777 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5032064535795878, + "low": 0.4451143466815108, + "mean": 0.4732314948732859, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5101833736846243, + "low": 0.5086662709327343, + "mean": 0.5093892624055697, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.5093892624055697, + "per_seed": { + "1073161778/train0": 0.5050884488962025, + "1073161778/train1": 0.506364326571218, + "1073161778/train2": 0.5038865127934079, + "1113371261/train0": 0.5130393809952023, + "1113371261/train1": 0.5096998710858021, + "1113371261/train2": 0.5110306378253284, + "1151410346/train0": 0.511440126674405, + "1151410346/train1": 0.5034160561308088, + "1151410346/train2": 0.5050707909765347, + "1261930456/train0": 0.506033274055822, + "1261930456/train1": 0.5043609091246408, + "1261930456/train2": 0.5048228430200322, + "1324133122/train0": 0.5161397587413304, + "1324133122/train1": 0.5152712504056621, + "1324133122/train2": 0.510107847781337, + "1324642411/train0": 0.5048732080022066, + "1324642411/train1": 0.507157276642487, + "1324642411/train2": 0.5096750531281539, + "1375492786/train0": 0.5080201327824784, + "1375492786/train1": 0.5124704817994062, + "1375492786/train2": 0.5044970161528288, + "141984615/train0": 0.5087967782363918, + "141984615/train1": 0.5148179816508425, + "141984615/train2": 0.5149845754360756, + "1461069713/train0": 0.5088280313121364, + "1461069713/train1": 0.5019504846479711, + "1461069713/train2": 0.511070602144867, + "1480856810/train0": 0.5065305341473837, + "1480856810/train1": 0.5135431948467644, + "1480856810/train2": 0.5049749254678583, + "1509980664/train0": 0.5045165731476825, + "1509980664/train1": 0.5059429296866168, + "1509980664/train2": 0.5075650128039051, + "1522272686/train0": 0.5320529802017496, + "1522272686/train1": 0.5150111595103795, + "1522272686/train2": 0.5129514889290346, + "1549324805/train0": 0.5144342311662078, + "1549324805/train1": 0.5137996065010031, + "1549324805/train2": 0.5129772970585401, + "1578990937/train0": 0.5133443564097337, + "1578990937/train1": 0.5081272058598021, + "1578990937/train2": 0.5092591404892353, + "1790188787/train0": 0.5036856098684014, + "1790188787/train1": 0.506979467301554, + "1790188787/train2": 0.5077554541172227, + "1822210471/train0": 0.5076183712392905, + "1822210471/train1": 0.5092194643264373, + "1822210471/train2": 0.5053265663635883, + "1845489013/train0": 0.507729136499722, + "1845489013/train1": 0.5041162134354321, + "1845489013/train2": 0.5045771512016884, + "1999022574/train0": 0.5113663967558075, + "1999022574/train1": 0.5079505274739976, + "1999022574/train2": 0.5098924941980084, + "2012211287/train0": 0.5025021287537501, + "2012211287/train1": 0.5023600590177417, + "2012211287/train2": 0.5075402669960806, + "2038594191/train0": 0.5141674398337075, + "2038594191/train1": 0.5220880545588878, + "2038594191/train2": 0.5174229574741631, + "2054899126/train0": 0.5050441593186851, + "2054899126/train1": 0.5093597426064234, + "2054899126/train2": 0.5093497250220118, + "2061635888/train0": 0.5121903376361622, + "2061635888/train1": 0.5130608055518229, + "2061635888/train2": 0.5104133544298581, + "211245842/train0": 0.5075662677800439, + "211245842/train1": 0.5069815835180018, + "211245842/train2": 0.5062349767915747, + "278008524/train0": 0.5092617090990478, + "278008524/train1": 0.5137154678536687, + "278008524/train2": 0.5125993969646946, + "307068879/train0": 0.5097271036094443, + "307068879/train1": 0.506525070880443, + "307068879/train2": 0.5115356866600222, + "412102563/train0": 0.5074318377589786, + "412102563/train1": 0.5071759937249536, + "412102563/train2": 0.5068073932444507, + "440577711/train0": 0.505089963282911, + "440577711/train1": 0.5069423941005992, + "441046872/train0": 0.5109682345626951, + "441046872/train1": 0.5075221409445348, + "441046872/train2": 0.5114229219309399, + "509430392/train0": 0.5079336789116323, + "509430392/train1": 0.5053331100145789, + "509430392/train2": 0.5049476605435552, + "585904130/train0": 0.5089117415058044, + "585904130/train1": 0.5082300568513344, + "585904130/train2": 0.5086682417290903, + "608016290/train0": 0.5065245373000641, + "608016290/train1": 0.5094605835953956, + "608016290/train2": 0.5060318822435502, + "626486813/train0": 0.5106419900796797, + "626486813/train1": 0.5087840592094554, + "626486813/train2": 0.5148650274185249, + "636680137/train0": 0.5126032019004112, + "636680137/train1": 0.5068342927158962, + "636680137/train2": 0.5084592529747467, + "655884605/train0": 0.5096348011231283, + "655884605/train1": 0.5078234193563246, + "655884605/train2": 0.5075199240973761, + "672328262/train0": 0.5094622393113466, + "672328262/train1": 0.5192789991993754, + "672328262/train2": 0.5164354775378074, + "701667486/train0": 0.5064903449632235, + "701667486/train1": 0.5093624359297296, + "701667486/train2": 0.5090635477222879, + "747260437/train0": 0.5094045097956438, + "747260437/train1": 0.5141512545538647, + "747260437/train2": 0.5072221251259937, + "750448296/train0": 0.5112937351142309, + "750448296/train1": 0.5143759815185674, + "750448296/train2": 0.5151650026688167, + "767747424/train0": 0.5102674583469905, + "767747424/train1": 0.5085176696480473, + "767747424/train2": 0.5082878761126821 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5155792047481198, + "low": 0.5074238891831525, + "mean": 0.5107026139222308, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5107026139222308, + "per_task": { + "LiftPegUpright-v1": 0.5071639936503661, + "PickCube-v1": 0.5200052095470546, + "PullCube-v1": 0.510716401449069, + "PushCube-v1": 0.506508150386812, + "StackCube-v1": 0.509119314577852 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5103089210843517, + "low": 0.5088787363107596, + "mean": 0.5095995072140308, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14969591549452674, + "low": 0.11426972935892137, + "mean": 0.13115703168394616, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.13115703168394616, + "per_seed": { + "1073161778/train0": 0.07651702032477398, + "1073161778/train1": 0.0650642448573534, + "1073161778/train2": 0.0038865127934079036, + "1113371261/train0": 0.08446795242377375, + "1113371261/train1": 0.13315727177134085, + "1113371261/train2": 0.15388778068247128, + "1151410346/train0": 0.2742741590398807, + "1151410346/train1": 0.21086965815490555, + "1151410346/train2": 0.13778635188060817, + "1261930456/train0": 0.2796810116584637, + "1261930456/train1": 0.04007519483892652, + "1261930456/train2": 0.10232001412282488, + "1324133122/train0": 0.1232826158841876, + "1324133122/train1": 0.19384267897709068, + "1324133122/train2": 0.3315364192099084, + "1324642411/train0": 0.14773035085934944, + "1324642411/train1": 0.2849350544202648, + "1324642411/train2": 0.009675053128153865, + "1375492786/train0": 0.43109705585940145, + "1375492786/train1": 0.04818476751369194, + "1375492786/train2": 0.07592558758140028, + "141984615/train0": 0.1697746503350368, + "141984615/train1": 0.09232487549201462, + "141984615/train2": 0.4078417182932184, + "1461069713/train0": 0.044542317026422085, + "1461069713/train1": 0.2519504846479711, + "1461069713/train2": 0.011070602144866992, + "1480856810/train0": 0.006530534147383693, + "1480856810/train1": 0.12068605198962151, + "1480856810/train2": 0.14783206832500112, + "1509980664/train0": 0.2554834268523175, + "1509980664/train1": 0.10594292968661678, + "1509980664/train2": 0.1352921300532378, + "1522272686/train0": 0.353481551630321, + "1522272686/train1": 0.050725445224665155, + "1522272686/train2": 0.08438006035760609, + "1549324805/train0": 0.2355657688337922, + "1549324805/train1": 0.1290575363561398, + "1549324805/train2": 0.012977297058540116, + "1578990937/train0": 0.16522707216169485, + "1578990937/train1": 0.008127205859802089, + "1578990937/train2": 0.04497342620352096, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.21060010441731292, + "1790188787/train1": 0.13587767555558894, + "1790188787/train2": 0.31042636406459556, + "1822210471/train0": 0.0076183712392905445, + "1822210471/train1": 0.009219464326437277, + "1822210471/train2": 0.041040852077874, + "1845489013/train0": 0.007729136499721956, + "1845489013/train1": 0.0041162134354321145, + "1845489013/train2": 0.21886286548740275, + "1999022574/train0": 0.18993782532723608, + "1999022574/train1": 0.09919232966885949, + "1999022574/train2": 0.3313210656265798, + "2012211287/train0": 0.07393070018232156, + "2012211287/train1": 0.0579156145732973, + "2012211287/train2": 0.20674544728963373, + "2038594191/train0": 0.19273886840513604, + "2038594191/train1": 0.04934051686968355, + "2038594191/train2": 0.23170867175987742, + "2054899126/train0": 0.020379222516485385, + "2054899126/train1": 0.11182272844606883, + "2054899126/train2": 0.04889656606042264, + "2061635888/train0": 0.2621903376361622, + "2061635888/train1": 0.19163223412325142, + "2061635888/train2": 0.010413354429858135, + "211245842/train0": 0.02814801793424182, + "211245842/train1": 0.06444698791056958, + "211245842/train2": 0.1009078803512824, + "278008524/train0": 0.004630854549523888, + "278008524/train1": 0.08242798035887994, + "278008524/train2": 0.13655744437479556, + "307068879/train0": 0.11686996075230144, + "307068879/train1": 0.04223935659472866, + "307068879/train2": 0.29724997237430795, + "412102563/train0": 0.1354253050981643, + "412102563/train1": 0.11431885086781074, + "412102563/train2": 0.18537882181587922, + "440577711/train0": 0.2806243224313747, + "440577711/train1": 0.3530962402544453, + "440577711/train2": 0.0, + "441046872/train0": 0.18953966313412368, + "441046872/train1": 0.1503792838016777, + "441046872/train2": 0.22570863621665416, + "509430392/train0": 0.06793367891163232, + "509430392/train1": 0.3235149281963971, + "509430392/train2": 0.23222038781628251, + "585904130/train0": 0.008911741505804427, + "585904130/train1": 0.07965862827990583, + "585904130/train2": 0.36581109887194746, + "608016290/train0": 0.006524537300064082, + "608016290/train1": 0.2594605835953956, + "608016290/train2": 0.13682526061359268, + "626486813/train0": 0.08207056150825115, + "626486813/train1": 0.12622228836950544, + "626486813/train2": 0.05656354401004643, + "636680137/train0": 0.1302539409567317, + "636680137/train1": 0.11397714985875335, + "636680137/train2": 0.0629693184538247, + "655884605/train0": 0.15249194398027116, + "655884605/train1": 0.04353770507061028, + "655884605/train2": 0.2932342098116618, + "672328262/train0": 0.009462239311346599, + "672328262/train1": 0.05214957222919603, + "672328262/train2": 0.12357833468066454, + "701667486/train0": 0.02922394075106216, + "701667486/train1": 0.08079100735830108, + "701667486/train2": 0.0980793094205692, + "747260437/train0": 0.02630977591864192, + "747260437/train1": 0.3072773168747067, + "747260437/train2": 0.1500792679831366, + "750448296/train0": 0.10872260477338204, + "750448296/train1": 0.0794795089520711, + "750448296/train2": 0.08645725719928145, + "767747424/train0": 0.19138713591745032, + "767747424/train1": 0.1598680319327742, + "767747424/train2": 0.12020697415901359 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.16055002015227218, + "low": 0.12324065494515828, + "mean": 0.14259542329562064, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.14259542329562064, + "per_task": { + "LiftPegUpright-v1": 0.12725711962690892, + "PickCube-v1": 0.1628623524041974, + "PullCube-v1": 0.10869460089822837, + "PushCube-v1": 0.1696532797298677, + "StackCube-v1": 0.14450976381890085 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.14442458910703884, + "low": 0.11201318885231094, + "mean": 0.12758416777878256, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 26.716666666666665, + "low": 24.608333333333334, + "mean": 25.747916666666665, + "n": 120, + "num_samples": 1000 + }, + "mean": 25.747916666666665, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 28.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 27.0, + "1324642411/train2": 18.0, + "1375492786/train0": 13.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 28.0, + "1480856810/train1": 28.0, + "1480856810/train2": 28.0, + "1509980664/train0": 25.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 28.0, + "1790188787/train1": 28.0, + "1790188787/train2": 22.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 28.0, + "1999022574/train1": 28.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 27.0, + "2012211287/train2": 28.0, + "2038594191/train0": 28.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 14.0, + "278008524/train1": 14.0, + "278008524/train2": 14.0, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 28.0, + "412102563/train1": 28.0, + "412102563/train2": 28.0, + "440577711/train0": 28.0, + "440577711/train1": 13.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 25.0, + "509430392/train1": 22.0, + "509430392/train2": 22.0, + "585904130/train0": 28.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 28.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 28.0, + "672328262/train1": 28.0, + "672328262/train2": 28.0, + "701667486/train0": 28.0, + "701667486/train1": 28.0, + "701667486/train2": 28.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 28.0, + "750448296/train1": 28.0, + "750448296/train2": 28.0, + "767747424/train0": 22.75, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 27.927272727272726, + "low": 24.69563636363636, + "mean": 26.348969696969696, + "n": 5, + "num_samples": 1000 + }, + "mean": 26.348969696969696, + "per_task": { + "LiftPegUpright-v1": 27.933333333333334, + "PickCube-v1": 28.0, + "PullCube-v1": 24.16, + "PushCube-v1": 23.833333333333332, + "StackCube-v1": 27.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 26.604166666666668, + "low": 24.104166666666668, + "mean": 25.430555555555557, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1702941699978705, + "low": 0.1349887469020671, + "mean": 0.1523158467647538, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.1523158467647538, + "per_seed": { + "1073161778/train0": 0.12685629258227518, + "1073161778/train1": 0.1717951848740798, + "1073161778/train2": 0.06826458778698574, + "1113371261/train0": 0.1065139428446398, + "1113371261/train1": 0.07977563885669328, + "1113371261/train2": 0.0920288600650524, + "1151410346/train0": 0.11740056999820042, + "1151410346/train1": 0.08293417068727207, + "1151410346/train2": 0.1106374656498088, + "1261930456/train0": 0.20374180832437194, + "1261930456/train1": 0.2409286430045787, + "1261930456/train2": 0.1368899848611159, + "1324133122/train0": 0.1385357960706925, + "1324133122/train1": 0.1825637783093442, + "1324133122/train2": 0.12394704754390919, + "1324642411/train0": 0.21794545086954079, + "1324642411/train1": 0.347647788530772, + "1324642411/train2": 0.14571765838194953, + "1375492786/train0": 0.016525300307342672, + "1375492786/train1": 0.19880639201897846, + "1375492786/train2": 0.08992311034993285, + "141984615/train0": 0.15640249398420367, + "141984615/train1": 0.16955591458280203, + "141984615/train2": 0.15743680246179145, + "1461069713/train0": 0.11446801340756732, + "1461069713/train1": 0.09666305797857698, + "1461069713/train2": 0.25927594010053956, + "1480856810/train0": 0.2674046465805726, + "1480856810/train1": 0.3753954390883053, + "1480856810/train2": 0.23055093221445233, + "1509980664/train0": 0.09788706834443102, + "1509980664/train1": 0.14589984808245227, + "1509980664/train2": 0.21186473269066222, + "1522272686/train0": 0.49245171577741104, + "1522272686/train1": 0.3716076139049983, + "1522272686/train2": 0.3918486005289839, + "1549324805/train0": 0.20313397288962703, + "1549324805/train1": 0.3351387925434473, + "1549324805/train2": 0.21570629252906995, + "1578990937/train0": 0.11674964401723253, + "1578990937/train1": 0.07562416744203525, + "1578990937/train2": 0.1280718516828302, + "1747178030/train0": 0.2736338609992832, + "1747178030/train1": 0.37283460103555477, + "1747178030/train2": 0.3199392510777099, + "1790188787/train0": 0.20654914374778957, + "1790188787/train1": 0.22799459496707442, + "1790188787/train2": 0.2395575582689906, + "1822210471/train0": 0.022563707465889905, + "1822210471/train1": 0.016662843715525302, + "1822210471/train2": -0.02439268371011942, + "1845489013/train0": 0.07034244814348488, + "1845489013/train1": 0.03038327939178103, + "1845489013/train2": -0.022266377420460608, + "1999022574/train0": 0.0923468998673774, + "1999022574/train1": 0.09800642070089008, + "1999022574/train2": 0.16067966181951557, + "2012211287/train0": 0.12745811668995927, + "2012211287/train1": 0.1339440868297946, + "2012211287/train2": 0.13101225256780796, + "2038594191/train0": 0.04432455250231335, + "2038594191/train1": 0.11815756078684783, + "2038594191/train2": 0.030673233140460502, + "2054899126/train0": 0.23480308650709372, + "2054899126/train1": 0.2679775511784741, + "2054899126/train2": 0.2067364502594559, + "2061635888/train0": 0.026008561069899745, + "2061635888/train1": 0.1199851428584793, + "2061635888/train2": 0.11936751939751099, + "211245842/train0": 0.14765513870898905, + "211245842/train1": 0.11500634632910782, + "211245842/train2": 0.17147291875581502, + "278008524/train0": 0.0642526633566908, + "278008524/train1": 0.052202521899224995, + "278008524/train2": 0.1449065418260213, + "307068879/train0": 0.17403822005730368, + "307068879/train1": 0.15887380976711568, + "307068879/train2": 0.18354981650868932, + "412102563/train0": 0.12581173470043255, + "412102563/train1": 0.11502862125762614, + "412102563/train2": 0.15894963977787818, + "440577711/train0": 0.28011676269294883, + "440577711/train1": 0.39808073658745674, + "440577711/train2": 0.30762508813304046, + "441046872/train0": 0.16069187570998772, + "441046872/train1": 0.13932622413588797, + "441046872/train2": 0.15405027288626078, + "509430392/train0": 0.265503929801179, + "509430392/train1": 0.2742130213071048, + "509430392/train2": 0.20165171424330564, + "585904130/train0": 0.024574618327409238, + "585904130/train1": 0.04771232196356422, + "585904130/train2": 0.0138091278166185, + "608016290/train0": 0.09283658594391792, + "608016290/train1": 0.16705607246529852, + "608016290/train2": 0.04324215665774371, + "626486813/train0": 0.05123780084861135, + "626486813/train1": 0.07804445205808362, + "626486813/train2": 0.07240150905862193, + "636680137/train0": -0.005906212010217932, + "636680137/train1": 0.01736105409945201, + "636680137/train2": 0.04118746392624513, + "655884605/train0": 0.0803760660625997, + "655884605/train1": 0.07436604773890146, + "655884605/train2": 0.09985821555853253, + "672328262/train0": 0.008566982900256922, + "672328262/train1": 0.16735307792538862, + "672328262/train2": -0.0007002341456026143, + "701667486/train0": 0.08941960203365407, + "701667486/train1": 0.1566669781571995, + "701667486/train2": 0.2544048252784154, + "747260437/train0": 0.19002955609852606, + "747260437/train1": 0.2604839026968317, + "747260437/train2": 0.1954184269801334, + "750448296/train0": 0.11891709722209572, + "750448296/train1": 0.18687518228997166, + "750448296/train2": 0.21615212478454096, + "767747424/train0": 0.17846547401748994, + "767747424/train1": 0.1775602262728388, + "767747424/train2": 0.1293231736933677 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.32627020569056, + "low": 0.12953357587420924, + "mean": 0.21279745256100208, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.21279745256100208, + "per_task": { + "LiftPegUpright-v1": 0.13553303703087385, + "PickCube-v1": 0.41863597673713104, + "PullCube-v1": 0.13437515079559892, + "PushCube-v1": 0.2537508278669194, + "StackCube-v1": 0.12169227037448728 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.16695848031863847, + "low": 0.13636356594482027, + "mean": 0.15243333068963227, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.29583333333333334, + "low": 0.15416666666666667, + "mean": 0.22291666666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.22291666666666668, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40606060606060607, + "low": 0.15903030303030302, + "mean": 0.2754747474747475, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2754747474747475, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.3333333333333333, + "PullCube-v1": 0.17333333333333334, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.18181818181818182 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.2916666666666667, + "low": 0.1597222222222222, + "mean": 0.2222222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7715499536289523, + "low": 0.5113214392965246, + "mean": 0.6364790620334033, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6364790620334033, + "per_seed": { + "1073161778/train0": 0.7180102467536926, + "1073161778/train1": 0.7134289145469666, + "1073161778/train2": 2.0, + "1113371261/train0": 0.0740400105714798, + "1113371261/train1": 0.09622243046760559, + "1113371261/train2": 0.1003647893667221, + "1151410346/train0": 0.5684944987297058, + "1151410346/train1": 0.2439272701740265, + "1151410346/train2": 0.5630103945732117, + "1261930456/train0": 0.09678153693675995, + "1261930456/train1": 0.09997129440307617, + "1261930456/train2": 0.08545441925525665, + "1324133122/train0": 0.0628519058227539, + "1324133122/train1": 0.0843733698129654, + "1324133122/train2": 0.09815146028995514, + "1324642411/train0": 0.21000581979751587, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.14743654429912567, + "1375492786/train2": 0.19452734291553497, + "141984615/train0": 0.023719489574432373, + "141984615/train1": 0.04397805407643318, + "141984615/train2": 0.04447716474533081, + "1461069713/train0": 0.05616961047053337, + "1461069713/train1": 0.05644957348704338, + "1461069713/train2": 0.11542954295873642, + "1480856810/train0": 0.14178013801574707, + "1480856810/train1": 2.0, + "1480856810/train2": 0.19622710347175598, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.83889240026474, + "1522272686/train0": 0.5233199596405029, + "1522272686/train1": 0.5422300696372986, + "1522272686/train2": 2.0, + "1549324805/train0": 0.21791090071201324, + "1549324805/train1": 0.3088321089744568, + "1549324805/train2": 0.21403594315052032, + "1578990937/train0": 0.21425709128379822, + "1578990937/train1": 0.21763037145137787, + "1578990937/train2": 0.23259200155735016, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.1862330436706543, + "1790188787/train2": 2.0, + "1822210471/train0": 0.03802649304270744, + "1822210471/train1": 0.07539409399032593, + "1822210471/train2": 0.058819372206926346, + "1845489013/train0": 0.0428844690322876, + "1845489013/train1": 0.09529292583465576, + "1845489013/train2": 0.0861951932311058, + "1999022574/train0": 0.5970457196235657, + "1999022574/train1": 2.0, + "1999022574/train2": 0.7970014810562134, + "2012211287/train0": 0.7277753949165344, + "2012211287/train1": 2.0, + "2012211287/train2": 0.7270426154136658, + "2038594191/train0": 0.15247176587581635, + "2038594191/train1": 0.1902807205915451, + "2038594191/train2": 0.24880830943584442, + "2054899126/train0": 1.0726270377635956, + "2054899126/train1": 1.0755530372262, + "2054899126/train2": 1.0683744624257088, + "2061635888/train0": 0.07642464339733124, + "2061635888/train1": 0.09046469628810883, + "2061635888/train2": 0.07902413606643677, + "211245842/train0": 0.5822281837463379, + "211245842/train1": 0.2016763538122177, + "211245842/train2": 0.2142850160598755, + "278008524/train0": 1.0722030997276306, + "278008524/train1": 1.1246033310890198, + "278008524/train2": 1.157032236456871, + "307068879/train0": 0.20280277729034424, + "307068879/train1": 0.20222051441669464, + "307068879/train2": 0.2035302072763443, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 0.8275061845779419, + "440577711/train0": 0.2057151049375534, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.203963503241539, + "441046872/train1": 0.20609354972839355, + "441046872/train2": 0.2042645812034607, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 0.16448359191417694, + "585904130/train1": 0.15698926150798798, + "585904130/train2": 0.2938849627971649, + "608016290/train0": 0.6124615669250488, + "608016290/train1": 0.6101007461547852, + "608016290/train2": 0.5705108642578125, + "626486813/train0": 0.05774742551147938, + "626486813/train1": 0.0619367565959692, + "626486813/train2": 0.08006523177027702, + "636680137/train0": 0.10778365284204483, + "636680137/train1": 0.11253269761800766, + "636680137/train2": 0.1247115358710289, + "655884605/train0": 0.15739890933036804, + "655884605/train1": 0.15038728713989258, + "655884605/train2": 0.17242512106895447, + "672328262/train0": 0.05979333445429802, + "672328262/train1": 0.053388457745313644, + "672328262/train2": 0.07216622680425644, + "701667486/train0": 0.7879213094711304, + "701667486/train1": 0.7947455048561096, + "701667486/train2": 2.0, + "747260437/train0": 0.19230616092681885, + "747260437/train1": 0.1342230886220932, + "747260437/train2": 0.18527252972126007, + "750448296/train0": 0.20943730572859445, + "750448296/train1": 0.22532296180725098, + "750448296/train2": 0.2481476068496704, + "767747424/train0": 0.5789013803005219, + "767747424/train1": 0.5825987188145518, + "767747424/train2": 0.5809952197596431 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.021850009759267, + "low": 0.5703265610120514, + "mean": 0.7913498574552512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7913498574552512, + "per_task": { + "LiftPegUpright-v1": 0.6601470321416855, + "PickCube-v1": 1.021850009759267, + "PullCube-v1": 0.46170220340291657, + "PushCube-v1": 1.1790093589160178, + "StackCube-v1": 0.634040683056369 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 0.7532538059684966, + "low": 0.5049100180880891, + "mean": 0.6232009150553495, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 3.548611111111111, + "low": 2.818055555555555, + "mean": 3.1958333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.1958333333333333, + "per_seed": { + "1073161778/train0": 4.0, + "1073161778/train1": 3.0, + "1073161778/train2": 2.0, + "1113371261/train0": 1.0, + "1113371261/train1": 3.0, + "1113371261/train2": 4.0, + "1151410346/train0": 3.0, + "1151410346/train1": 6.0, + "1151410346/train2": 7.0, + "1261930456/train0": 5.0, + "1261930456/train1": 3.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 6.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 1.0, + "1324642411/train2": 4.0, + "1375492786/train0": 2.0, + "1375492786/train1": 7.0, + "1375492786/train2": 6.0, + "141984615/train0": 4.0, + "141984615/train1": 1.0, + "141984615/train2": 6.0, + "1461069713/train0": 3.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 5.0, + "1509980664/train2": 5.0, + "1522272686/train0": 1.0, + "1522272686/train1": 5.0, + "1522272686/train2": 4.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 7.0, + "1578990937/train1": 6.0, + "1578990937/train2": 6.0, + "1747178030/train0": 1.0, + "1747178030/train1": 4.0, + "1747178030/train2": 6.0, + "1790188787/train0": 0.0, + "1790188787/train1": 4.0, + "1790188787/train2": 1.0, + "1822210471/train0": 3.0, + "1822210471/train1": 3.0, + "1822210471/train2": 2.0, + "1845489013/train0": 7.0, + "1845489013/train1": 1.0, + "1845489013/train2": 6.0, + "1999022574/train0": 2.0, + "1999022574/train1": 4.0, + "1999022574/train2": 4.0, + "2012211287/train0": 5.0, + "2012211287/train1": 1.0, + "2012211287/train2": 4.0, + "2038594191/train0": 4.0, + "2038594191/train1": 5.0, + "2038594191/train2": 1.0, + "2054899126/train0": 4.5, + "2054899126/train1": 3.0, + "2054899126/train2": 0.0, + "2061635888/train0": 2.0, + "2061635888/train1": 1.0, + "2061635888/train2": 4.0, + "211245842/train0": 3.0, + "211245842/train1": 6.0, + "211245842/train2": 2.0, + "278008524/train0": 2.0, + "278008524/train1": 4.0, + "278008524/train2": 3.0, + "307068879/train0": 3.0, + "307068879/train1": 4.0, + "307068879/train2": 5.0, + "412102563/train0": 3.0, + "412102563/train1": 1.0, + "412102563/train2": 4.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 2.0, + "441046872/train2": 1.0, + "509430392/train0": 2.0, + "509430392/train1": 0.0, + "509430392/train2": 5.0, + "585904130/train0": 7.0, + "585904130/train1": 4.0, + "585904130/train2": 0.0, + "608016290/train0": 5.0, + "608016290/train1": 1.0, + "608016290/train2": 2.0, + "626486813/train0": 6.5, + "626486813/train1": 4.0, + "626486813/train2": 2.0, + "636680137/train0": 1.0, + "636680137/train1": 6.0, + "636680137/train2": 7.0, + "655884605/train0": 6.0, + "655884605/train1": 5.0, + "655884605/train2": 1.0, + "672328262/train0": 5.0, + "672328262/train1": 2.0, + "672328262/train2": 7.0, + "701667486/train0": 7.0, + "701667486/train1": 4.0, + "701667486/train2": 7.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 4.0, + "750448296/train0": 1.3333333333333333, + "750448296/train1": 3.6666666666666665, + "750448296/train2": 3.0, + "767747424/train0": 4.5, + "767747424/train1": 3.5, + "767747424/train2": 3.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 3.5796363636363635, + "low": 2.4893333333333336, + "mean": 3.059434343434343, + "n": 5, + "num_samples": 1000 + }, + "mean": 3.059434343434343, + "per_task": { + "LiftPegUpright-v1": 2.8, + "PickCube-v1": 3.3333333333333335, + "PullCube-v1": 3.28, + "PushCube-v1": 1.9444444444444444, + "StackCube-v1": 3.9393939393939394 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.576388888888889, + "low": 2.8402777777777777, + "mean": 3.2152777777777777, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3875, + "low": 0.21805555555555556, + "mean": 0.30000000000000004, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30000000000000004, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6515151515151516, + "low": 0.10836363636363637, + "mean": 0.39018181818181813, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.39018181818181813, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.6666666666666666, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.4027777777777778, + "low": 0.25, + "mean": 0.3263888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.025, + "mean": -0.008333333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.008333333333333333, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": -1.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.2, + "mean": -0.06666666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.06666666666666667, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": -0.3333333333333333, + "PullCube-v1": 0.0, + "PushCube-v1": 0.0, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.0, + "low": -0.020833333333333332, + "mean": -0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9397181576531793, + "low": 0.6461840466285745, + "mean": 0.7872578506978848, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7872578506978848, + "per_seed": { + "1073161778/train0": 0.725517988204956, + "1073161778/train1": 0.725517988204956, + "1073161778/train2": 0.725517988204956, + "1113371261/train0": 0.11109928786754608, + "1113371261/train1": 0.11109928786754608, + "1113371261/train2": 0.11109928786754608, + "1151410346/train0": 0.2284735143184662, + "1151410346/train1": 0.2284735143184662, + "1151410346/train2": 0.2284735143184662, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.084227055311203, + "1324133122/train2": 0.084227055311203, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.16783273220062256, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.22852759063243866, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 0.13452494144439697, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.17249532043933868, + "1549324805/train1": 0.17249532043933868, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21426597237586975, + "1578990937/train1": 0.21426597237586975, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.09442546218633652, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.09442546218633652, + "1845489013/train0": 0.11255800724029541, + "1845489013/train1": 0.11255800724029541, + "1845489013/train2": 0.11255800724029541, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.09504059702157974, + "2038594191/train1": 0.09504059702157974, + "2038594191/train2": 0.09504059702157974, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.08620692789554596, + "2061635888/train1": 0.08620692789554596, + "2061635888/train2": 0.08620692789554596, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.1250072121620178, + "278008524/train1": 1.1250072121620178, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.20915235579013824, + "307068879/train1": 0.20915235579013824, + "307068879/train2": 0.20915235579013824, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.2097676396369934, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2097676396369934, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.07103653997182846, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.07103653997182846, + "608016290/train0": 0.525360107421875, + "608016290/train1": 0.525360107421875, + "608016290/train2": 0.525360107421875, + "626486813/train0": 0.04174089431762695, + "626486813/train1": 0.04174089431762695, + "626486813/train2": 0.04174089431762695, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.1440753936767578, + "655884605/train2": 0.1440753936767578, + "672328262/train0": 0.10593423992395401, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.10593423992395401, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7605033268531164, + "750448296/train1": 0.7605033268531164, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0747087560594082, + "767747424/train1": 1.0747087560594082, + "767747424/train2": 1.0747087560594082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3797807363398147, + "low": 0.5526382644826715, + "mean": 0.9669703430063799, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9669703430063799, + "per_task": { + "LiftPegUpright-v1": 0.4796089559793472, + "PickCube-v1": 1.3781749804814656, + "PullCube-v1": 0.8032506120204925, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.5003613992170854 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.9544860299987098, + "low": 0.6815054596712192, + "mean": 0.8208790636724896, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.04663687646389007, + "mean": -0.01554562548796336, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.01554562548796336, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": -1.865475058555603, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.3730950117111206, + "mean": -0.12436500390370686, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.12436500390370686, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": -0.6218250195185343, + "PullCube-v1": 0.0, + "PushCube-v1": 0.0, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.0, + "low": -0.038864063719908394, + "mean": -0.012954687906636132, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.21380325423864027, + "low": 0.07175614044277205, + "mean": 0.1359608985690607, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.1359608985690607, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.274482011795044, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.3400209844112396, + "1151410346/train1": 0.015453755855560303, + "1151410346/train2": 0.3345368802547455, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.00014631450176239014, + "1324133122/train2": 0.013924404978752136, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.167920172214508, + "1509980664/train1": 1.167920172214508, + "1509980664/train2": 0.006812572479248047, + "1522272686/train0": 0.38879501819610596, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.04541558027267456, + "1549324805/train1": 0.1363367885351181, + "1549324805/train2": 0.04154062271118164, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0033643990755081177, + "1578990937/train2": 0.018326029181480408, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.1720311045646667, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.2717533111572266, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0574311688542366, + "2038594191/train1": 0.09524012356996536, + "2038594191/train2": 0.15376771241426468, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.004257768392562866, + "2061635888/train2": 0.0, + "211245842/train0": 0.3979237377643585, + "211245842/train1": 0.017371907830238342, + "211245842/train2": 0.029980570077896118, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.03202502429485321, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.9592653959989548, + "509430392/train1": 1.9592653959989548, + "509430392/train2": 1.9592653959989548, + "585904130/train0": 0.09344705194234848, + "585904130/train1": 0.08595272153615952, + "585904130/train2": 0.22284842282533646, + "608016290/train0": 0.08710145950317383, + "608016290/train1": 0.08474063873291016, + "608016290/train2": 0.0451507568359375, + "626486813/train0": 0.016006531193852425, + "626486813/train1": 0.020195862278342247, + "626486813/train2": 0.03832433745265007, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.005137942731380463, + "655884605/train0": 0.01332351565361023, + "655884605/train1": 0.006311893463134766, + "655884605/train2": 0.028349727392196655, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0034714341163635254, + "701667486/train2": 1.208725929260254, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.06344164411226909, + "750448296/train1": 0.06390197078386943, + "750448296/train2": 0.11175449689229329, + "767747424/train0": 0.019455675035715103, + "767747424/train1": 0.019207116216421127, + "767747424/train2": 0.013884376734495163 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2586892498532931, + "low": 0.08747102950016657, + "mean": 0.16967375873255006, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.16967375873255006, + "per_task": { + "LiftPegUpright-v1": 0.18421521186828613, + "PickCube-v1": 0.129598339398702, + "PullCube-v1": 0.027702728708585103, + "PushCube-v1": 0.32654423266649246, + "StackCube-v1": 0.18030828102068466 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.1848001828055001, + "low": 0.061897138625176415, + "mean": 0.11845623132669264, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.125, + "low": 0.03333333333333333, + "mean": 0.075, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.075, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.14181818181818182, + "low": 0.024242424242424242, + "mean": 0.08424242424242424, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.08424242424242424, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.10416666666666667, + "low": 0.027777777777777776, + "mean": 0.0625, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6597222222222222, + "low": 0.4847222222222222, + "mean": 0.5708333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5708333333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8072727272727273, + "low": 0.40533333333333327, + "mean": 0.6063030303030303, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6063030303030303, + "per_task": { + "LiftPegUpright-v1": 0.8666666666666667, + "PickCube-v1": 0.6666666666666666, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.8181818181818182 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.5972222222222222, + "low": 0.4375, + "mean": 0.5138888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1496533814558965, + "low": 0.8442967605000982, + "mean": 0.9982878092624661, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.9982878092624661, + "per_seed": { + "1073161778/train0": 1.2819897532463074, + "1073161778/train1": 1.2865710854530334, + "1073161778/train2": 0.0, + "1113371261/train0": 1.9259599894285202, + "1113371261/train1": 1.9037775695323944, + "1113371261/train2": 1.899635210633278, + "1151410346/train0": 1.4315055012702942, + "1151410346/train1": 1.7560727298259735, + "1151410346/train2": 1.4369896054267883, + "1261930456/train0": 1.90321846306324, + "1261930456/train1": 1.9000287055969238, + "1261930456/train2": 1.9145455807447433, + "1324133122/train0": 1.937148094177246, + "1324133122/train1": 1.9156266301870346, + "1324133122/train2": 1.9018485397100449, + "1324642411/train0": 1.7899941802024841, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.8525634557008743, + "1375492786/train2": 1.805472657084465, + "141984615/train0": 0.2620577812194824, + "141984615/train1": 0.2417992167174816, + "141984615/train2": 0.24130010604858398, + "1461069713/train0": 0.10317284241318703, + "1461069713/train1": 0.10289287939667702, + "1461069713/train2": 0.04391290992498398, + "1480856810/train0": 0.06770022213459015, + "1480856810/train1": 0.0, + "1480856810/train2": 0.013253256678581238, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.16110759973526, + "1522272686/train0": 1.476680040359497, + "1522272686/train1": 1.4577699303627014, + "1522272686/train2": 0.0, + "1549324805/train0": 0.07954518496990204, + "1549324805/train1": 0.0, + "1549324805/train2": 0.08342014253139496, + "1578990937/train0": 1.7857429087162018, + "1578990937/train1": 1.7823696285486221, + "1578990937/train2": 1.7674079984426498, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8137669563293457, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9619735069572926, + "1822210471/train1": 1.924605906009674, + "1822210471/train2": 1.9411806277930737, + "1845489013/train0": 1.9571155309677124, + "1845489013/train1": 1.9047070741653442, + "1845489013/train2": 1.9138048067688942, + "1999022574/train0": 1.4029542803764343, + "1999022574/train1": 0.0, + "1999022574/train2": 1.2029985189437866, + "2012211287/train0": 1.2722246050834656, + "2012211287/train1": 0.0, + "2012211287/train2": 1.2729573845863342, + "2038594191/train0": 1.8475282341241837, + "2038594191/train1": 1.809719279408455, + "2038594191/train2": 1.7511916905641556, + "2054899126/train0": 0.9273729622364044, + "2054899126/train1": 0.9244469627737999, + "2054899126/train2": 0.9316255375742912, + "2061635888/train0": 1.9235753566026688, + "2061635888/train1": 1.9095353037118912, + "2061635888/train2": 1.9209758639335632, + "211245842/train0": 1.417771816253662, + "211245842/train1": 1.7983236461877823, + "211245842/train2": 1.7857149839401245, + "278008524/train0": 0.04170554131269455, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.7971972227096558, + "307068879/train1": 1.7977794855833054, + "307068879/train2": 1.7964697927236557, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.172493815422058, + "440577711/train0": 1.7942848950624466, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.796036496758461, + "441046872/train1": 1.7939064502716064, + "441046872/train2": 1.7957354187965393, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.835516408085823, + "585904130/train1": 1.843010738492012, + "585904130/train2": 1.706115037202835, + "608016290/train0": 1.3875384330749512, + "608016290/train1": 1.3898992538452148, + "608016290/train2": 1.4294891357421875, + "626486813/train0": 0.1078349556773901, + "626486813/train1": 0.10364562459290028, + "626486813/train2": 0.08551714941859245, + "636680137/train0": 0.06278067082166672, + "636680137/train1": 0.05803162604570389, + "636680137/train2": 0.04585278779268265, + "655884605/train0": 1.842601090669632, + "655884605/train1": 1.8496127128601074, + "655884605/train2": 1.8275748789310455, + "672328262/train0": 1.940206665545702, + "672328262/train1": 1.9466115422546864, + "672328262/train2": 1.9278337731957436, + "701667486/train0": 1.2120786905288696, + "701667486/train1": 1.2052544951438904, + "701667486/train2": 0.0, + "747260437/train0": 0.08635708689689636, + "747260437/train1": 0.144440159201622, + "747260437/train2": 0.09339071810245514, + "750448296/train0": 1.1985591898361843, + "750448296/train1": 1.1875135501225789, + "750448296/train2": 1.1598488887151082, + "767747424/train0": 0.10475751012563705, + "767747424/train1": 0.1008116127923131, + "767747424/train2": 0.09709237236529589 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2779549119689249, + "low": 0.7197053071856498, + "mean": 0.9988301095772872, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9988301095772872, + "per_task": { + "LiftPegUpright-v1": 1.3398529678583144, + "PickCube-v1": 0.9781499902407328, + "PullCube-v1": 0.6881442584097386, + "PushCube-v1": 0.6220440144340197, + "StackCube-v1": 1.365959316943631 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 1.045588894850678, + "low": 0.7773104406013671, + "mean": 0.9091424554741631, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.41382592280226776, + "low": 0.41382592280226776, + "mean": 0.41382592280226776, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.41382592280226776, + "per_seed": { + "1073161778/train0": 0.41382592280226843, + "1073161778/train1": 0.41382592280226843, + "1073161778/train2": 0.41382592280226843, + "1113371261/train0": 0.41382592280226843, + "1113371261/train1": 0.41382592280226843, + "1113371261/train2": 0.41382592280226843, + "1151410346/train0": 0.41382592280226843, + "1151410346/train1": 0.41382592280226843, + "1151410346/train2": 0.41382592280226843, + "1261930456/train0": 0.41382592280226843, + "1261930456/train1": 0.41382592280226843, + "1261930456/train2": 0.41382592280226843, + "1324133122/train0": 0.41382592280226843, + "1324133122/train1": 0.41382592280226843, + "1324133122/train2": 0.41382592280226843, + "1324642411/train0": 0.41382592280226843, + "1324642411/train1": 0.41382592280226843, + "1324642411/train2": 0.41382592280226843, + "1375492786/train0": 0.41382592280226843, + "1375492786/train1": 0.41382592280226843, + "1375492786/train2": 0.41382592280226843, + "141984615/train0": 0.41382592280226843, + "141984615/train1": 0.41382592280226843, + "141984615/train2": 0.41382592280226843, + "1461069713/train0": 0.41382592280226843, + "1461069713/train1": 0.41382592280226843, + "1461069713/train2": 0.41382592280226843, + "1480856810/train0": 0.41382592280226843, + "1480856810/train1": 0.41382592280226843, + "1480856810/train2": 0.41382592280226843, + "1509980664/train0": 0.41382592280226843, + "1509980664/train1": 0.41382592280226843, + "1509980664/train2": 0.41382592280226843, + "1522272686/train0": 0.41382592280226843, + "1522272686/train1": 0.41382592280226843, + "1522272686/train2": 0.41382592280226843, + "1549324805/train0": 0.41382592280226843, + "1549324805/train1": 0.41382592280226843, + "1549324805/train2": 0.41382592280226843, + "1578990937/train0": 0.41382592280226843, + "1578990937/train1": 0.41382592280226843, + "1578990937/train2": 0.41382592280226843, + "1747178030/train0": 0.41382592280226843, + "1747178030/train1": 0.41382592280226843, + "1747178030/train2": 0.41382592280226843, + "1790188787/train0": 0.41382592280226843, + "1790188787/train1": 0.41382592280226843, + "1790188787/train2": 0.41382592280226843, + "1822210471/train0": 0.41382592280226843, + "1822210471/train1": 0.41382592280226843, + "1822210471/train2": 0.41382592280226843, + "1845489013/train0": 0.41382592280226843, + "1845489013/train1": 0.41382592280226843, + "1845489013/train2": 0.41382592280226843, + "1999022574/train0": 0.41382592280226843, + "1999022574/train1": 0.41382592280226843, + "1999022574/train2": 0.41382592280226843, + "2012211287/train0": 0.41382592280226843, + "2012211287/train1": 0.41382592280226843, + "2012211287/train2": 0.41382592280226843, + "2038594191/train0": 0.41382592280226843, + "2038594191/train1": 0.41382592280226843, + "2038594191/train2": 0.41382592280226843, + "2054899126/train0": 0.41382592280226843, + "2054899126/train1": 0.41382592280226843, + "2054899126/train2": 0.41382592280226843, + "2061635888/train0": 0.41382592280226843, + "2061635888/train1": 0.41382592280226843, + "2061635888/train2": 0.41382592280226843, + "211245842/train0": 0.41382592280226843, + "211245842/train1": 0.41382592280226843, + "211245842/train2": 0.41382592280226843, + "278008524/train0": 0.41382592280226843, + "278008524/train1": 0.41382592280226843, + "278008524/train2": 0.41382592280226843, + "307068879/train0": 0.41382592280226843, + "307068879/train1": 0.41382592280226843, + "307068879/train2": 0.41382592280226843, + "412102563/train0": 0.41382592280226843, + "412102563/train1": 0.41382592280226843, + "412102563/train2": 0.41382592280226843, + "440577711/train0": 0.41382592280226843, + "440577711/train1": 0.41382592280226843, + "440577711/train2": 0.41382592280226843, + "441046872/train0": 0.41382592280226843, + "441046872/train1": 0.41382592280226843, + "441046872/train2": 0.41382592280226843, + "509430392/train0": 0.41382592280226843, + "509430392/train1": 0.41382592280226843, + "509430392/train2": 0.41382592280226843, + "585904130/train0": 0.41382592280226843, + "585904130/train1": 0.41382592280226843, + "585904130/train2": 0.41382592280226843, + "608016290/train0": 0.41382592280226843, + "608016290/train1": 0.41382592280226843, + "608016290/train2": 0.41382592280226843, + "626486813/train0": 0.41382592280226843, + "626486813/train1": 0.41382592280226843, + "626486813/train2": 0.41382592280226843, + "636680137/train0": 0.41382592280226843, + "636680137/train1": 0.41382592280226843, + "636680137/train2": 0.41382592280226843, + "655884605/train0": 0.41382592280226843, + "655884605/train1": 0.41382592280226843, + "655884605/train2": 0.41382592280226843, + "672328262/train0": 0.41382592280226843, + "672328262/train1": 0.41382592280226843, + "672328262/train2": 0.41382592280226843, + "701667486/train0": 0.41382592280226843, + "701667486/train1": 0.41382592280226843, + "701667486/train2": 0.41382592280226843, + "747260437/train0": 0.41382592280226843, + "747260437/train1": 0.41382592280226843, + "747260437/train2": 0.41382592280226843, + "750448296/train0": 0.41382592280226843, + "750448296/train1": 0.41382592280226843, + "750448296/train2": 0.41382592280226843, + "767747424/train0": 0.41382592280226843, + "767747424/train1": 0.41382592280226843, + "767747424/train2": 0.41382592280226843 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4138259228022684, + "low": 0.41382592280226793, + "mean": 0.4138259228022682, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4138259228022682, + "per_task": { + "LiftPegUpright-v1": 0.41382592280226843, + "PickCube-v1": 0.41382592280226843, + "PullCube-v1": 0.4138259228022678, + "PushCube-v1": 0.41382592280226843, + "StackCube-v1": 0.4138259228022681 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.41382592280226776, + "low": 0.41382592280226776, + "mean": 0.41382592280226776, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "success", + "tau": 0.41382592280226843, + "threshold_scope": "global", + "weights": [ + 0.10807291666666696, + -0.04498870245455899, + 0.046530481708683256, + 0.010589731566355804, + 0.00979141868860928, + -0.0031617827934025252, + -0.02191180624286721, + -0.0687335545608507, + 0.12479390234544024, + 1.1288146367454118e-17, + 0.0030654003930455625, + 0.00011001751201419972, + 0.003369291733439906, + -0.007794381079125551, + 0.003855206771244733, + -0.006064736999548532, + -0.005675599749051325, + 0.003101571820539053, + 0.0030654003930455057, + 0.00011001751201408956, + 0.0033692917334399316, + -0.007794381079125478, + 0.003855206771244788, + -0.006064736999548527, + -0.0056755997490513, + 0.003101571820539077, + 0.004505842590307981, + 0.00323404209060182, + -0.0020845331228138016, + -0.003813539686393111, + 0.0016007965529255081, + 0.0006604989896916592, + -0.005623504244012833, + 0.004170464843451778, + 0.0, + -0.003382457091382061, + -0.0038450997588419947 + ] +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..bc1bae170dbc5334220fa6e5db0ad82c81dcb6c7 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics_by_seed.json @@ -0,0 +1,3234 @@ +{ + "1073161778/train0": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5050884488962025, + "pairwise_causal_calibration_ece": 0.07651702032477398, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12685629258227518, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "tau": 0.41382592280226843 + }, + "1073161778/train1": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.506364326571218, + "pairwise_causal_calibration_ece": 0.0650642448573534, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1717951848740798, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "tau": 0.41382592280226843 + }, + "1073161778/train2": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5038865127934079, + "pairwise_causal_calibration_ece": 0.0038865127934079036, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06826458778698574, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274482011795044, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5130393809952023, + "pairwise_causal_calibration_ece": 0.08446795242377375, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1065139428446398, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "tau": 0.41382592280226843 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5096998710858021, + "pairwise_causal_calibration_ece": 0.13315727177134085, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07977563885669328, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "tau": 0.41382592280226843 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5110306378253284, + "pairwise_causal_calibration_ece": 0.15388778068247128, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0920288600650524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "tau": 0.41382592280226843 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.511440126674405, + "pairwise_causal_calibration_ece": 0.2742741590398807, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11740056999820042, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3400209844112396, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "tau": 0.41382592280226843 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5034160561308088, + "pairwise_causal_calibration_ece": 0.21086965815490555, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08293417068727207, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.015453755855560303, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "tau": 0.41382592280226843 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5050707909765347, + "pairwise_causal_calibration_ece": 0.13778635188060817, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1106374656498088, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3345368802547455, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "tau": 0.41382592280226843 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.506033274055822, + "pairwise_causal_calibration_ece": 0.2796810116584637, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20374180832437194, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "tau": 0.41382592280226843 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5043609091246408, + "pairwise_causal_calibration_ece": 0.04007519483892652, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2409286430045787, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "tau": 0.41382592280226843 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5048228430200322, + "pairwise_causal_calibration_ece": 0.10232001412282488, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1368899848611159, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "tau": 0.41382592280226843 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5161397587413304, + "pairwise_causal_calibration_ece": 0.1232826158841876, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1385357960706925, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "tau": 0.41382592280226843 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5152712504056621, + "pairwise_causal_calibration_ece": 0.19384267897709068, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1825637783093442, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.00014631450176239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "tau": 0.41382592280226843 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.510107847781337, + "pairwise_causal_calibration_ece": 0.3315364192099084, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12394704754390919, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013924404978752136, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "tau": 0.41382592280226843 + }, + "1324642411/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5048732080022066, + "pairwise_causal_calibration_ece": 0.14773035085934944, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21794545086954079, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "tau": 0.41382592280226843 + }, + "1324642411/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.507157276642487, + "pairwise_causal_calibration_ece": 0.2849350544202648, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.347647788530772, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1324642411/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5096750531281539, + "pairwise_causal_calibration_ece": 0.009675053128153865, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": 0.14571765838194953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07692307692307693, + "pairwise_causal_calibration_confidence": 0.5080201327824784, + "pairwise_causal_calibration_ece": 0.43109705585940145, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.016525300307342672, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5124704817994062, + "pairwise_causal_calibration_ece": 0.04818476751369194, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19880639201897846, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "tau": 0.41382592280226843 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5044970161528288, + "pairwise_causal_calibration_ece": 0.07592558758140028, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08992311034993285, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "tau": 0.41382592280226843 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5087967782363918, + "pairwise_causal_calibration_ece": 0.1697746503350368, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15640249398420367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "tau": 0.41382592280226843 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5148179816508425, + "pairwise_causal_calibration_ece": 0.09232487549201462, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16955591458280203, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "tau": 0.41382592280226843 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5149845754360756, + "pairwise_causal_calibration_ece": 0.4078417182932184, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15743680246179145, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "tau": 0.41382592280226843 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5088280313121364, + "pairwise_causal_calibration_ece": 0.044542317026422085, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11446801340756732, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "tau": 0.41382592280226843 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5019504846479711, + "pairwise_causal_calibration_ece": 0.2519504846479711, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09666305797857698, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "tau": 0.41382592280226843 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.511070602144867, + "pairwise_causal_calibration_ece": 0.011070602144866992, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25927594010053956, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "tau": 0.41382592280226843 + }, + "1480856810/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5065305341473837, + "pairwise_causal_calibration_ece": 0.006530534147383693, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2674046465805726, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "tau": 0.41382592280226843 + }, + "1480856810/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5135431948467644, + "pairwise_causal_calibration_ece": 0.12068605198962151, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3753954390883053, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1480856810/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5049749254678583, + "pairwise_causal_calibration_ece": 0.14783206832500112, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.23055093221445233, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "tau": 0.41382592280226843 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.76, + "pairwise_causal_calibration_confidence": 0.5045165731476825, + "pairwise_causal_calibration_ece": 0.2554834268523175, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.09788706834443102, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5059429296866168, + "pairwise_causal_calibration_ece": 0.10594292968661678, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.14589984808245227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5075650128039051, + "pairwise_causal_calibration_ece": 0.1352921300532378, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21186473269066222, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "tau": 0.41382592280226843 + }, + "1522272686/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5320529802017496, + "pairwise_causal_calibration_ece": 0.353481551630321, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.49245171577741104, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13452494144439697, + "selected_utility_gain_over_base": -1.865475058555603, + "selector_regret": 0.38879501819610596, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "tau": 0.41382592280226843 + }, + "1522272686/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5150111595103795, + "pairwise_causal_calibration_ece": 0.050725445224665155, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3716076139049983, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "tau": 0.41382592280226843 + }, + "1522272686/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5129514889290346, + "pairwise_causal_calibration_ece": 0.08438006035760609, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3918486005289839, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5144342311662078, + "pairwise_causal_calibration_ece": 0.2355657688337922, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20313397288962703, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04541558027267456, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "tau": 0.41382592280226843 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5137996065010031, + "pairwise_causal_calibration_ece": 0.1290575363561398, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3351387925434473, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1363367885351181, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5129772970585401, + "pairwise_causal_calibration_ece": 0.012977297058540116, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21570629252906995, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04154062271118164, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "tau": 0.41382592280226843 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5133443564097337, + "pairwise_causal_calibration_ece": 0.16522707216169485, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11674964401723253, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "tau": 0.41382592280226843 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5081272058598021, + "pairwise_causal_calibration_ece": 0.008127205859802089, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07562416744203525, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0033643990755081177, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "tau": 0.41382592280226843 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5092591404892353, + "pairwise_causal_calibration_ece": 0.04497342620352096, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1280718516828302, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018326029181480408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "tau": 0.41382592280226843 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.2736338609992832, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.37283460103555477, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.3199392510777099, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1790188787/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5036856098684014, + "pairwise_causal_calibration_ece": 0.21060010441731292, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20654914374778957, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1790188787/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.506979467301554, + "pairwise_causal_calibration_ece": 0.13587767555558894, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22799459496707442, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "tau": 0.41382592280226843 + }, + "1790188787/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8181818181818182, + "pairwise_causal_calibration_confidence": 0.5077554541172227, + "pairwise_causal_calibration_ece": 0.31042636406459556, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.2395575582689906, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5076183712392905, + "pairwise_causal_calibration_ece": 0.0076183712392905445, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.022563707465889905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "tau": 0.41382592280226843 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5092194643264373, + "pairwise_causal_calibration_ece": 0.009219464326437277, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.016662843715525302, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "tau": 0.41382592280226843 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5053265663635883, + "pairwise_causal_calibration_ece": 0.041040852077874, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.02439268371011942, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "tau": 0.41382592280226843 + }, + "1845489013/train0": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.507729136499722, + "pairwise_causal_calibration_ece": 0.007729136499721956, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07034244814348488, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "tau": 0.41382592280226843 + }, + "1845489013/train1": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5041162134354321, + "pairwise_causal_calibration_ece": 0.0041162134354321145, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03038327939178103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "tau": 0.41382592280226843 + }, + "1845489013/train2": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5045771512016884, + "pairwise_causal_calibration_ece": 0.21886286548740275, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.022266377420460608, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "tau": 0.41382592280226843 + }, + "1999022574/train0": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5113663967558075, + "pairwise_causal_calibration_ece": 0.18993782532723608, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0923468998673774, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "tau": 0.41382592280226843 + }, + "1999022574/train1": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5079505274739976, + "pairwise_causal_calibration_ece": 0.09919232966885949, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09800642070089008, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "1999022574/train2": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5098924941980084, + "pairwise_causal_calibration_ece": 0.3313210656265798, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16067966181951557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "tau": 0.41382592280226843 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5025021287537501, + "pairwise_causal_calibration_ece": 0.07393070018232156, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12745811668995927, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "tau": 0.41382592280226843 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5023600590177417, + "pairwise_causal_calibration_ece": 0.0579156145732973, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.1339440868297946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2717533111572266, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5075402669960806, + "pairwise_causal_calibration_ece": 0.20674544728963373, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13101225256780796, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "tau": 0.41382592280226843 + }, + "2038594191/train0": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5141674398337075, + "pairwise_causal_calibration_ece": 0.19273886840513604, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04432455250231335, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0574311688542366, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "tau": 0.41382592280226843 + }, + "2038594191/train1": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5220880545588878, + "pairwise_causal_calibration_ece": 0.04934051686968355, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11815756078684783, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09524012356996536, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "tau": 0.41382592280226843 + }, + "2038594191/train2": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5174229574741631, + "pairwise_causal_calibration_ece": 0.23170867175987742, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.030673233140460502, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.15376771241426468, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "tau": 0.41382592280226843 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5050441593186851, + "pairwise_causal_calibration_ece": 0.020379222516485385, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.23480308650709372, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0726270377635956, + "selected_candidate_index": 4.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9273729622364044, + "tau": 0.41382592280226843 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5093597426064234, + "pairwise_causal_calibration_ece": 0.11182272844606883, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.2679775511784741, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0755530372262, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9244469627737999, + "tau": 0.41382592280226843 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5093497250220118, + "pairwise_causal_calibration_ece": 0.04889656606042264, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.2067364502594559, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0683744624257088, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9316255375742912, + "tau": 0.41382592280226843 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5121903376361622, + "pairwise_causal_calibration_ece": 0.2621903376361622, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.026008561069899745, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "tau": 0.41382592280226843 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5130608055518229, + "pairwise_causal_calibration_ece": 0.19163223412325142, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1199851428584793, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.004257768392562866, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "tau": 0.41382592280226843 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5104133544298581, + "pairwise_causal_calibration_ece": 0.010413354429858135, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11936751939751099, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "tau": 0.41382592280226843 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5075662677800439, + "pairwise_causal_calibration_ece": 0.02814801793424182, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14765513870898905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3979237377643585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "tau": 0.41382592280226843 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5069815835180018, + "pairwise_causal_calibration_ece": 0.06444698791056958, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11500634632910782, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.017371907830238342, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "tau": 0.41382592280226843 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5062349767915747, + "pairwise_causal_calibration_ece": 0.1009078803512824, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17147291875581502, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.029980570077896118, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "tau": 0.41382592280226843 + }, + "278008524/train0": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5092617090990478, + "pairwise_causal_calibration_ece": 0.004630854549523888, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.0642526633566908, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0722030997276306, + "selected_candidate_index": 2.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04170554131269455, + "tau": 0.41382592280226843 + }, + "278008524/train1": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5137154678536687, + "pairwise_causal_calibration_ece": 0.08242798035887994, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.052202521899224995, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1246033310890198, + "selected_candidate_index": 4.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "278008524/train2": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5125993969646946, + "pairwise_causal_calibration_ece": 0.13655744437479556, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.1449065418260213, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.157032236456871, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03202502429485321, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5097271036094443, + "pairwise_causal_calibration_ece": 0.11686996075230144, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17403822005730368, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "tau": 0.41382592280226843 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.506525070880443, + "pairwise_causal_calibration_ece": 0.04223935659472866, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15887380976711568, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "tau": 0.41382592280226843 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5115356866600222, + "pairwise_causal_calibration_ece": 0.29724997237430795, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18354981650868932, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "tau": 0.41382592280226843 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5074318377589786, + "pairwise_causal_calibration_ece": 0.1354253050981643, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12581173470043255, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5071759937249536, + "pairwise_causal_calibration_ece": 0.11431885086781074, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11502862125762614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5068073932444507, + "pairwise_causal_calibration_ece": 0.18537882181587922, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15894963977787818, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "tau": 0.41382592280226843 + }, + "440577711/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.505089963282911, + "pairwise_causal_calibration_ece": 0.2806243224313747, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.28011676269294883, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "tau": 0.41382592280226843 + }, + "440577711/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.15384615384615385, + "pairwise_causal_calibration_confidence": 0.5069423941005992, + "pairwise_causal_calibration_ece": 0.3530962402544453, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.39808073658745674, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "440577711/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.30762508813304046, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5109682345626951, + "pairwise_causal_calibration_ece": 0.18953966313412368, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16069187570998772, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "tau": 0.41382592280226843 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5075221409445348, + "pairwise_causal_calibration_ece": 0.1503792838016777, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13932622413588797, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "tau": 0.41382592280226843 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5114229219309399, + "pairwise_causal_calibration_ece": 0.22570863621665416, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15405027288626078, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "tau": 0.41382592280226843 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44, + "pairwise_causal_calibration_confidence": 0.5079336789116323, + "pairwise_causal_calibration_ece": 0.06793367891163232, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.265503929801179, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.18181818181818182, + "pairwise_causal_calibration_confidence": 0.5053331100145789, + "pairwise_causal_calibration_ece": 0.3235149281963971, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.2742130213071048, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2727272727272727, + "pairwise_causal_calibration_confidence": 0.5049476605435552, + "pairwise_causal_calibration_ece": 0.23222038781628251, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.20165171424330564, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5089117415058044, + "pairwise_causal_calibration_ece": 0.008911741505804427, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.024574618327409238, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09344705194234848, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "tau": 0.41382592280226843 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082300568513344, + "pairwise_causal_calibration_ece": 0.07965862827990583, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04771232196356422, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08595272153615952, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "tau": 0.41382592280226843 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5086682417290903, + "pairwise_causal_calibration_ece": 0.36581109887194746, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0138091278166185, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.22284842282533646, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "tau": 0.41382592280226843 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5065245373000641, + "pairwise_causal_calibration_ece": 0.006524537300064082, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09283658594391792, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08710145950317383, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "tau": 0.41382592280226843 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5094605835953956, + "pairwise_causal_calibration_ece": 0.2594605835953956, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16705607246529852, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08474063873291016, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "tau": 0.41382592280226843 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5060318822435502, + "pairwise_causal_calibration_ece": 0.13682526061359268, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04324215665774371, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0451507568359375, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "tau": 0.41382592280226843 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5106419900796797, + "pairwise_causal_calibration_ece": 0.08207056150825115, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05123780084861135, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05774742551147938, + "selected_candidate_index": 6.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04174089431762695, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.016006531193852425, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1078349556773901, + "tau": 0.41382592280226843 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4107142857142857, + "pairwise_causal_calibration_confidence": 0.5087840592094554, + "pairwise_causal_calibration_ece": 0.12622228836950544, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07804445205808362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0619367565959692, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04174089431762695, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.020195862278342247, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10364562459290028, + "tau": 0.41382592280226843 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5148650274185249, + "pairwise_causal_calibration_ece": 0.05656354401004643, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07240150905862193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08006523177027702, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04174089431762695, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03832433745265007, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08551714941859245, + "tau": 0.41382592280226843 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5126032019004112, + "pairwise_causal_calibration_ece": 0.1302539409567317, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.005906212010217932, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "tau": 0.41382592280226843 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5068342927158962, + "pairwise_causal_calibration_ece": 0.11397714985875335, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.01736105409945201, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "tau": 0.41382592280226843 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5084592529747467, + "pairwise_causal_calibration_ece": 0.0629693184538247, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04118746392624513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005137942731380463, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "tau": 0.41382592280226843 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5096348011231283, + "pairwise_causal_calibration_ece": 0.15249194398027116, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0803760660625997, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01332351565361023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "tau": 0.41382592280226843 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5078234193563246, + "pairwise_causal_calibration_ece": 0.04353770507061028, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07436604773890146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006311893463134766, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "tau": 0.41382592280226843 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5075199240973761, + "pairwise_causal_calibration_ece": 0.2932342098116618, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09985821555853253, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.028349727392196655, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "tau": 0.41382592280226843 + }, + "672328262/train0": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5094622393113466, + "pairwise_causal_calibration_ece": 0.009462239311346599, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.008566982900256922, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "tau": 0.41382592280226843 + }, + "672328262/train1": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5192789991993754, + "pairwise_causal_calibration_ece": 0.05214957222919603, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16735307792538862, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "tau": 0.41382592280226843 + }, + "672328262/train2": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5164354775378074, + "pairwise_causal_calibration_ece": 0.12357833468066454, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0007002341456026143, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "tau": 0.41382592280226843 + }, + "701667486/train0": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5064903449632235, + "pairwise_causal_calibration_ece": 0.02922394075106216, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08941960203365407, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "tau": 0.41382592280226843 + }, + "701667486/train1": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5093624359297296, + "pairwise_causal_calibration_ece": 0.08079100735830108, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1566669781571995, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "tau": 0.41382592280226843 + }, + "701667486/train2": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5090635477222879, + "pairwise_causal_calibration_ece": 0.0980793094205692, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2544048252784154, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.41382592280226843 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5094045097956438, + "pairwise_causal_calibration_ece": 0.02630977591864192, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19002955609852606, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "tau": 0.41382592280226843 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5141512545538647, + "pairwise_causal_calibration_ece": 0.3072773168747067, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2604839026968317, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "tau": 0.41382592280226843 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5072221251259937, + "pairwise_causal_calibration_ece": 0.1500792679831366, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1954184269801334, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "tau": 0.41382592280226843 + }, + "750448296/train0": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.5119047619047619, + "pairwise_causal_calibration_confidence": 0.5112937351142309, + "pairwise_causal_calibration_ece": 0.10872260477338204, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11891709722209572, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20943730572859445, + "selected_candidate_index": 1.3333333333333333, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06344164411226909, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1985591898361843, + "tau": 0.41382592280226843 + }, + "750448296/train1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5143759815185674, + "pairwise_causal_calibration_ece": 0.0794795089520711, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18687518228997166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22532296180725098, + "selected_candidate_index": 3.6666666666666665, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06390197078386943, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1875135501225789, + "tau": 0.41382592280226843 + }, + "750448296/train2": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.5476190476190476, + "pairwise_causal_calibration_confidence": 0.5151650026688167, + "pairwise_causal_calibration_ece": 0.08645725719928145, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21615212478454096, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2481476068496704, + "selected_candidate_index": 3.0, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.11175449689229329, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1598488887151082, + "tau": 0.41382592280226843 + }, + "767747424/train0": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.4017857142857143, + "pairwise_causal_calibration_confidence": 0.5102674583469905, + "pairwise_causal_calibration_ece": 0.19138713591745032, + "pairwise_causal_calibration_pairs": 22.75, + "predicted_margin": 0.17846547401748994, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5789013803005219, + "selected_candidate_index": 4.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0747087560594082, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019455675035715103, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10475751012563705, + "tau": 0.41382592280226843 + }, + "767747424/train1": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.4285714285714286, + "pairwise_causal_calibration_confidence": 0.5085176696480473, + "pairwise_causal_calibration_ece": 0.1598680319327742, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1775602262728388, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5825987188145518, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0747087560594082, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019207116216421127, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1008116127923131, + "tau": 0.41382592280226843 + }, + "767747424/train2": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.5892857142857143, + "pairwise_causal_calibration_confidence": 0.5082878761126821, + "pairwise_causal_calibration_ece": 0.12020697415901359, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1293231736933677, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5809952197596431, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0747087560594082, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013884376734495163, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09709237236529589, + "tau": 0.41382592280226843 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..4e4399fcbbc79f1ecfd5a809f1120c7ff59fe14d --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.0, + "base_utility": 0.4796089559793472, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.4343915343915344, + "pairwise_causal_calibration_confidence": 0.5071639936503661, + "pairwise_causal_calibration_ece": 0.12725711962690892, + "pairwise_causal_calibration_pairs": 27.933333333333334, + "predicted_margin": 0.13553303703087385, + "proposal_oracle_success": 0.13333333333333333, + "proposal_oracle_utility": 0.6601470321416855, + "selected_candidate_index": 2.8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.4796089559793472, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.18421521186828613, + "success_selector_gap": 0.13333333333333333, + "success_support_gap": 0.8666666666666667, + "support_gap": 1.3398529678583144, + "tau": 0.41382592280226843 + }, + "PickCube-v1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5200052095470546, + "pairwise_causal_calibration_ece": 0.1628623524041974, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.41863597673713104, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 1.021850009759267, + "selected_candidate_index": 3.3333333333333335, + "selected_success": 0.6666666666666666, + "selected_success_gain_over_base": -0.3333333333333333, + "selected_utility": 1.3781749804814656, + "selected_utility_gain_over_base": -0.6218250195185343, + "selector_regret": 0.129598339398702, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 0.9781499902407328, + "tau": 0.41382592280226843 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.8032506120204925, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.32, + "pairwise_causal_calibration_accuracy": 0.48276723276723277, + "pairwise_causal_calibration_confidence": 0.510716401449069, + "pairwise_causal_calibration_ece": 0.10869460089822837, + "pairwise_causal_calibration_pairs": 24.16, + "predicted_margin": 0.13437515079559892, + "proposal_oracle_success": 0.17333333333333334, + "proposal_oracle_utility": 0.46170220340291657, + "selected_candidate_index": 3.28, + "selected_success": 0.36, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8032506120204925, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.027702728708585103, + "success_selector_gap": 0.0, + "success_support_gap": 0.3466666666666667, + "support_gap": 0.6881442584097386, + "tau": 0.4138259228022678 + }, + "PushCube-v1": { + "base_success": 0.8333333333333334, + "base_utility": 1.6734557673335075, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.16666666666666666, + "pairwise_causal_calibration_accuracy": 0.4821140297610885, + "pairwise_causal_calibration_confidence": 0.506508150386812, + "pairwise_causal_calibration_ece": 0.1696532797298677, + "pairwise_causal_calibration_pairs": 23.833333333333332, + "predicted_margin": 0.2537508278669194, + "proposal_oracle_success": 0.5555555555555556, + "proposal_oracle_utility": 1.1790093589160178, + "selected_candidate_index": 1.9444444444444444, + "selected_success": 0.8333333333333334, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.6734557673335075, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32654423266649246, + "success_selector_gap": 0.16666666666666666, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6220440144340197, + "tau": 0.41382592280226843 + }, + "StackCube-v1": { + "base_success": 0.09090909090909091, + "base_utility": 0.5003613992170854, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6060606060606061, + "pairwise_causal_calibration_accuracy": 0.47779220779220777, + "pairwise_causal_calibration_confidence": 0.509119314577852, + "pairwise_causal_calibration_ece": 0.14450976381890085, + "pairwise_causal_calibration_pairs": 27.818181818181817, + "predicted_margin": 0.12169227037448728, + "proposal_oracle_success": 0.18181818181818182, + "proposal_oracle_utility": 0.634040683056369, + "selected_candidate_index": 3.9393939393939394, + "selected_success": 0.09090909090909091, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5003613992170854, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.18030828102068466, + "success_selector_gap": 0.12121212121212122, + "success_support_gap": 0.8181818181818182, + "support_gap": 1.365959316943631, + "tau": 0.4138259228022681 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..4045815ee4136aa68bf6c8f94aa48c1ff8fca6ec --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.0069 & 0.9931 & 0.3333 & 0.3264 & 0.2222 & 0.3611 & 0.5139 & 0.0625 & 0.0315 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e7f624010a258c1868824fbf355089d3d92425bc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_perdim_trainmax_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=0.0 +tau=0.413826 +calibration_input=runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics.json index 8461573706f8eda5a7ef1e13f698f223f20dce3c..efd9864a06038a33923aee9b3d5a71856b570f09 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics.json @@ -26,6 +26,10 @@ "hidden_chart_oracle_success": 0.6597222222222222, "hidden_chart_oracle_utility": 1.4150809271054134, "outcome_ptr": 0.3773148148148148, + "pairwise_causal_calibration_accuracy": 0.49463779846216105, + "pairwise_causal_calibration_confidence": 0.5114127025750097, + "pairwise_causal_calibration_ece": 0.016774904112848654, + "pairwise_causal_calibration_pairs": 9884.0, "proposal_oracle_success": 0.36342592592592593, "proposal_oracle_utility": 0.8099956938270751, "selected_success": 0.36342592592592593, @@ -49,6 +53,10 @@ "hidden_chart_oracle_success": 0.7291666666666666, "hidden_chart_oracle_utility": 1.5183161981403828, "outcome_ptr": 0.4166666666666667, + "pairwise_causal_calibration_accuracy": 0.5066666666666667, + "pairwise_causal_calibration_confidence": 0.5121503411460236, + "pairwise_causal_calibration_ece": 0.005483674479356937, + "pairwise_causal_calibration_pairs": 3450.0, "proposal_oracle_success": 0.3819444444444444, "proposal_oracle_utility": 0.8643959594224321, "selected_success": 0.3819444444444444, @@ -179,11 +187,203 @@ 0.03103759015348356, 0.02828865844409604 ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 3456, + "num_fit_rows": 3456, + "num_pairwise_rows": 0, + "num_pointwise_rows": 3456, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", "k": 8, "num_calibration_candidates": 3456, "num_calibration_rows": 432, "num_eval_candidates": 1152, "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.49463779846216105, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.016774904112848654, + "accuracy": 0.49463779846216105, + "confidence": 0.5114127025750097, + "count": 9884, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.016774904112848654, + "mean_confidence": 0.5114127025750097, + "n_bins": 10, + "num_pairs": 9884, + "num_rows": 432 + }, + "eval": { + "accuracy": 0.5066666666666667, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.005483674479356937, + "accuracy": 0.5066666666666667, + "confidence": 0.5121503411460236, + "count": 3450, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.005483674479356937, + "mean_confidence": 0.5121503411460236, + "n_bins": 10, + "num_pairs": 3450, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, "report_type": "learned_dominance_selector_eval", "ridge_lambdas": [ 0.0, @@ -204,6 +404,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5079042501299456, + "pairwise_causal_calibration_ece": 0.027810035584340143, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20092600070845926, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -231,6 +435,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.517127112474433, + "pairwise_causal_calibration_ece": 0.05284139818871869, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21756370458471622, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20391923189163208, @@ -258,6 +466,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5098566341155361, + "pairwise_causal_calibration_ece": 0.009856634115536056, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.197805317803883, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -285,6 +497,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5093751939822863, + "pairwise_causal_calibration_ece": 0.13348194887485665, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1717547088237561, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -312,6 +528,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5192681731031695, + "pairwise_causal_calibration_ece": 0.23355388738888383, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2043317342807088, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5837622880935669, @@ -339,6 +559,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5250784055152783, + "pairwise_causal_calibration_ece": 0.09650697694384974, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3284611884666808, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16821590065956116, @@ -366,6 +590,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.29301731712998935, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -393,6 +621,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.08593665939740933, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -420,6 +652,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5097775167745581, + "pairwise_causal_calibration_ece": 0.00977751677455807, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1969950204810189, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06010149046778679, @@ -447,6 +683,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5077433261782961, + "pairwise_causal_calibration_ece": 0.07917189760686755, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14912474810327114, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.012636144645512104, @@ -474,6 +714,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5089660423581928, + "pairwise_causal_calibration_ece": 0.062462529070378636, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2545011031059808, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15609906613826752, @@ -501,6 +745,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5152932666576685, + "pairwise_causal_calibration_ece": 0.2119794606150588, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.0886735921808986, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -528,6 +776,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5077609250056717, + "pairwise_causal_calibration_ece": 0.277953360708614, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2718526065472824, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16806542873382568, @@ -555,6 +807,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.510692599528415, + "pairwise_causal_calibration_ece": 0.02502168618587075, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11485541576913086, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07040411233901978, @@ -582,6 +838,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5108692392445339, + "pairwise_causal_calibration_ece": 0.11801209638739102, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14108783056103189, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02055191993713379, @@ -609,6 +869,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5150319676147265, + "pairwise_causal_calibration_ece": 0.020682318099559227, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15128401783007617, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07324296236038208, @@ -636,6 +900,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5134139185128666, + "pairwise_causal_calibration_ece": 0.04912820422715225, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3322775859297912, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -663,6 +931,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5104367146945467, + "pairwise_causal_calibration_ece": 0.045118840861008924, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.09578732958371006, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -690,6 +962,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.507901687873458, + "pairwise_causal_calibration_ece": 0.04361597358774372, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20959728035122172, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.008802552707493305, @@ -717,6 +993,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5179157263010408, + "pairwise_causal_calibration_ece": 0.12494141655610214, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1478317876482174, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.23932616412639618, @@ -744,6 +1024,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5110307445144335, + "pairwise_causal_calibration_ece": 0.011030744514433533, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11748877632482482, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09099660068750381, @@ -771,6 +1055,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5061002778266522, + "pairwise_causal_calibration_ece": 0.10104257931620486, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.125609952965325, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.020476778969168663, @@ -798,6 +1086,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.1142441575920736, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -825,6 +1117,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5185185185185185, + "pairwise_causal_calibration_confidence": 0.520866835944586, + "pairwise_causal_calibration_ece": 0.0023483174260675588, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.11438425065440704, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -852,6 +1148,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5090362001405325, + "pairwise_causal_calibration_ece": 0.3838209427166104, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26834491061766463, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05724634602665901, @@ -879,6 +1179,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.29664492343304527, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -906,6 +1210,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5082874271968991, + "pairwise_causal_calibration_ece": 0.04400171291118482, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.22414376708358918, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13453228771686554, @@ -933,6 +1241,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5044689315862549, + "pairwise_causal_calibration_ece": 0.04992347704080041, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.10364268178154662, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -960,6 +1272,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082800251982494, + "pairwise_causal_calibration_ece": 0.0797085966268209, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23866768065123248, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04792773723602295, @@ -987,6 +1303,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5148884431169273, + "pairwise_causal_calibration_ece": 0.12203130025978443, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11067121611907713, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1014,6 +1334,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36363636363636365, + "pairwise_causal_calibration_confidence": 0.5074098621746328, + "pairwise_causal_calibration_ece": 0.14377349853826915, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.07680075864093804, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1041,6 +1365,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5052959089864312, + "pairwise_causal_calibration_ece": 0.3624387661292883, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.6870668145379031, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1068,6 +1396,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5088647318101668, + "pairwise_causal_calibration_ece": 0.2588647318101668, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6712258551783686, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.17992261052131653, @@ -1095,6 +1427,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5040985707207823, + "pairwise_causal_calibration_ece": 0.3612414278636395, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.6291681545836397, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1122,6 +1458,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5097695136941484, + "pairwise_causal_calibration_ece": 0.22405522797986266, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6193304427041512, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06612107157707214, @@ -1149,6 +1489,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5059629842951442, + "pairwise_causal_calibration_ece": 0.029751301419141507, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6471664922001317, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12932512164115906, @@ -1176,6 +1520,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5092043778610017, + "pairwise_causal_calibration_ece": 0.15746228880566493, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.6561771188937511, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1203,6 +1551,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5193863539393203, + "pairwise_causal_calibration_ece": 0.16224349679646316, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20663971147287907, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.0642656534910202, @@ -1230,6 +1582,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5196382790206474, + "pairwise_causal_calibration_ece": 0.2696382790206474, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.24752873446742565, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1631268858909607, @@ -1257,6 +1613,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5198284636786039, + "pairwise_causal_calibration_ece": 0.12697132082146106, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.246546487439865, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09061422944068909, @@ -1284,6 +1644,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5158496642258744, + "pairwise_causal_calibration_ece": 0.05557890720269698, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26276318940133603, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20236456394195557, @@ -1311,6 +1675,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5107026203890623, + "pairwise_causal_calibration_ece": 0.33213119181763373, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2595476416603391, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1338,6 +1706,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5093770540150734, + "pairwise_causal_calibration_ece": 0.062051517413498014, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2818040383848624, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1365,6 +1737,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5145081193786424, + "pairwise_causal_calibration_ece": 0.2712061663356433, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2766048493279699, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20478898286819458, @@ -1392,6 +1768,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5224226018004667, + "pairwise_causal_calibration_ece": 0.09385117322903819, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26820123426100245, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.056939899921417236, @@ -1419,6 +1799,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.511181830700846, + "pairwise_causal_calibration_ece": 0.2034895230085383, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.2713698658138815, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1446,6 +1830,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5110298208425068, + "pairwise_causal_calibration_ece": 0.36817267798536396, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.2782043046573905, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1473,6 +1861,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.24, + "pairwise_causal_calibration_confidence": 0.5186386660397809, + "pairwise_causal_calibration_ece": 0.27863866603978094, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.25444049345463426, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1500,6 +1892,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5038741047450654, + "pairwise_causal_calibration_ece": 0.06755446668350595, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16349292872459203, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1527,6 +1923,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5139732937786187, + "pairwise_causal_calibration_ece": 0.19254472235004721, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1801716769818562, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20632563531398773, @@ -1554,6 +1954,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5091112114195717, + "pairwise_causal_calibration_ece": 0.06231736000899968, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1847762400277908, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7075161337852478, @@ -1581,6 +1985,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3888888888888889, + "pairwise_causal_calibration_confidence": 0.5098943896149679, + "pairwise_causal_calibration_ece": 0.12100550072607902, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.20877294341834293, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1608,6 +2016,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5155934510473184, + "pairwise_causal_calibration_ece": 0.1941648796187469, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16033886964133004, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7209141850471497, @@ -1635,6 +2047,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5105908420311568, + "pairwise_causal_calibration_ece": 0.2394091579688432, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2760379294226463, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07200749218463898, @@ -1662,6 +2078,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.3803269800393895, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1689,6 +2109,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5065856139275622, + "pairwise_causal_calibration_ece": 0.14944275678470503, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12385745148209507, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14892418682575226, @@ -1716,6 +2140,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5045038789520366, + "pairwise_causal_calibration_ece": 0.004503878952036633, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20397936444755985, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.030005255714058876, @@ -1743,6 +2171,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5128099051893484, + "pairwise_causal_calibration_ece": 0.19138133376077698, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21054284458495792, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.01455958653241396, @@ -1770,6 +2202,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5076812671130492, + "pairwise_causal_calibration_ece": 0.17089016145837943, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.28425160955643003, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15548823773860931, @@ -1797,6 +2233,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5278734108303779, + "pairwise_causal_calibration_ece": 0.13879325583628876, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.18825713114760673, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1824,6 +2264,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5087029042218038, + "pairwise_causal_calibration_ece": 0.13415423863533915, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.32449166823206843, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.23258204758167267, @@ -1851,6 +2295,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5110663753955008, + "pairwise_causal_calibration_ece": 0.3103621960330706, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1317783678868279, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1599414199590683, @@ -1878,6 +2326,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5114463604438318, + "pairwise_causal_calibration_ece": 0.05998221098473955, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1226666983652919, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05916740372776985, @@ -1905,6 +2357,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5104495518828907, + "pairwise_causal_calibration_ece": 0.09669330525996644, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14649895141116653, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03854070231318474, @@ -1932,6 +2388,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5077859533325537, + "pairwise_causal_calibration_ece": 0.04350023904683942, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18953514564987203, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.010124921798706055, @@ -1959,6 +2419,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082796231877765, + "pairwise_causal_calibration_ece": 0.07970819461634798, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.10249988539655566, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1290181279182434, @@ -1986,6 +2450,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5164579122807452, + "pairwise_causal_calibration_ece": 0.16211351629068338, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21183340573001233, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03755499795079231, @@ -2013,6 +2481,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5147846951064375, + "pairwise_causal_calibration_ece": 0.30664387632213386, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18611879379280816, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18459641933441162, @@ -2040,6 +2512,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5087141999697643, + "pairwise_causal_calibration_ece": 0.027000085744521396, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1312853076823468, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08514270931482315, @@ -2067,6 +2543,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.505653484497765, + "pairwise_causal_calibration_ece": 0.11279634164062219, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16222791725929153, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04168079420924187, @@ -2094,6 +2574,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5070611656440998, + "pairwise_causal_calibration_ece": 0.007061165644099754, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1658630525898114, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09690030664205551, @@ -2121,6 +2605,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5261612748851421, + "pairwise_causal_calibration_ece": 0.045267296543429314, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17159672738556292, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15240873396396637, @@ -2148,6 +2636,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5139449923088458, + "pairwise_causal_calibration_ece": 0.09319786483401127, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.24932229999298094, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1424257904291153, @@ -2175,6 +2667,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.28397868346027655, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2202,6 +2698,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5105870204506016, + "pairwise_causal_calibration_ece": 0.13227012240654135, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27086090181173816, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06424945592880249, @@ -2229,6 +2729,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.508201282106844, + "pairwise_causal_calibration_ece": 0.02751300360744169, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12303133928452442, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2256,6 +2760,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5088781157128205, + "pairwise_causal_calibration_ece": 0.2588781157128205, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3111189505567141, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2170039862394333, @@ -2283,6 +2791,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5177759880984708, + "pairwise_causal_calibration_ece": 0.19634741666989936, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09669884825042896, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13508938252925873, @@ -2310,6 +2822,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.5114594422009907, + "pairwise_causal_calibration_ece": 0.203767134508683, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.1579275162186275, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2337,6 +2853,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 1.0, + "pairwise_causal_calibration_confidence": 0.5089048821442496, + "pairwise_causal_calibration_ece": 0.49109511785575044, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7609842811296632, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2364,6 +2884,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.508953471583655, + "pairwise_causal_calibration_ece": 0.06247509984491639, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7232270300064332, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1777031123638153, @@ -2391,6 +2915,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.6121226617843838, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2418,6 +2946,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5085443341947974, + "pairwise_causal_calibration_ece": 0.17002709437663122, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.667791884975374, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05957312881946564, @@ -2445,6 +2977,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9090909090909091, + "pairwise_causal_calibration_confidence": 0.5089098400968083, + "pairwise_causal_calibration_ece": 0.4001810689941008, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.6675461655230028, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2472,6 +3008,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.509309423579836, + "pairwise_causal_calibration_ece": 0.18788085215126454, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7595411023961984, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19969357550144196, @@ -2499,6 +3039,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5172736961846305, + "pairwise_causal_calibration_ece": 0.05298798189891618, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26282861689601783, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07914192974567413, @@ -2526,6 +3070,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5166281511611449, + "pairwise_causal_calibration_ece": 0.2309138654468592, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23791187673632117, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15427637100219727, @@ -2553,6 +3101,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5196427235751272, + "pairwise_causal_calibration_ece": 0.0517858478534442, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23873008123815037, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14092671871185303, @@ -2580,6 +3132,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5115243772227345, + "pairwise_causal_calibration_ece": 0.024189908491551226, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2479399896937514, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20639251172542572, @@ -2607,6 +3163,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36, + "pairwise_causal_calibration_confidence": 0.5100155711510672, + "pairwise_causal_calibration_ece": 0.15001557115106723, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.276906017761115, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2634,6 +3194,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5065274413865767, + "pairwise_causal_calibration_ece": 0.279186844327709, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27950117722622586, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22683429718017578, @@ -2661,6 +3225,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5069167821110796, + "pairwise_causal_calibration_ece": 0.1359403607460633, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23010369929396896, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2086353600025177, @@ -2688,6 +3256,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5227821807492782, + "pairwise_causal_calibration_ece": 0.04864639067929322, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.30466427900455206, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12235099077224731, @@ -2715,6 +3287,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5114740860753181, + "pairwise_causal_calibration_ece": 0.36861694321817523, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.30277388356719176, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2742,6 +3318,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.507869130131741, + "pairwise_causal_calibration_ece": 0.22215484441745526, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.2854092085136363, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2769,6 +3349,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4090909090909091, + "pairwise_causal_calibration_confidence": 0.5119334766574462, + "pairwise_causal_calibration_ece": 0.10284256756653704, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.24790697988984683, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2796,6 +3380,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5094648157635078, + "pairwise_causal_calibration_ece": 0.21780791150921952, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.17995610986243743, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2823,6 +3411,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5182153571387064, + "pairwise_causal_calibration_ece": 0.053929642852992066, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20469744001592355, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20428544282913208, @@ -2850,6 +3442,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6111111111111112, + "pairwise_causal_calibration_confidence": 0.5089181730493006, + "pairwise_causal_calibration_ece": 0.10219293806181051, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.14402410971307644, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2877,6 +3473,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5120271305387815, + "pairwise_causal_calibration_ece": 0.19059855911021, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13802477297102525, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664259791374207, @@ -2904,6 +3504,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5184906548624142, + "pairwise_causal_calibration_ece": 0.37563351200527134, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19688713080112988, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.21979393064975739, @@ -2931,6 +3535,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5231037401616476, + "pairwise_causal_calibration_ece": 0.05881802587593332, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3311257655653472, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15220001339912415, @@ -2958,6 +3566,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.3463013216157089, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2985,6 +3597,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5061999236378464, + "pairwise_causal_calibration_ece": 0.18477135220927493, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13947044060217628, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3012,6 +3628,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5110427839399747, + "pairwise_causal_calibration_ece": 0.1675286446314539, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27422519801828676, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09976043552160263, @@ -3039,6 +3659,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5146869710834582, + "pairwise_causal_calibration_ece": 0.05674160034511322, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16571024088016414, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03799549862742424, @@ -3066,6 +3690,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5094631537502677, + "pairwise_causal_calibration_ece": 0.1523202966074106, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.22574390322821564, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10632803291082382, @@ -3093,6 +3721,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.12, + "pairwise_causal_calibration_confidence": 0.5211353121802774, + "pairwise_causal_calibration_ece": 0.4011353121802774, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.08486096062823595, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3120,6 +3752,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5087800882875294, + "pairwise_causal_calibration_ece": 0.1340770545696135, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2690901197537252, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20969144999980927, @@ -3147,6 +3783,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9642857142857143, + "pairwise_causal_calibration_confidence": 0.5138366438250436, + "pairwise_causal_calibration_ece": 0.45044907046067073, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16685967997774187, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2908984124660492, @@ -3174,6 +3814,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5089957841341125, + "pairwise_causal_calibration_ece": 0.18756721270554105, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07991407615376411, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3201,6 +3845,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.513939558487466, + "pairwise_causal_calibration_ece": 0.09320329865539112, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12044529325726602, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.029790839180350304, @@ -3228,6 +3876,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5069525642466648, + "pairwise_causal_calibration_ece": 0.2073331500390495, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21796952105933293, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.007658918853849173, @@ -3255,6 +3907,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5113970881858346, + "pairwise_causal_calibration_ece": 0.024317197528451073, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09962893251516124, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3282,6 +3938,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5171743151724543, + "pairwise_causal_calibration_ece": 0.33860288660102567, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18308305510200482, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03313249349594116, @@ -3309,6 +3969,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5205340645267428, + "pairwise_causal_calibration_ece": 0.12232307833040013, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16199433700898458, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22729186713695526, @@ -3336,6 +4000,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5078943863682678, + "pairwise_causal_calibration_ece": 0.17067704220316082, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1345855010442043, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08911379426717758, @@ -3363,6 +4031,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5129361277059727, + "pairwise_causal_calibration_ece": 0.34420672943688435, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15001899961573703, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.053396303206682205, @@ -3390,6 +4062,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5065725175893429, + "pairwise_causal_calibration_ece": 0.1494296604464857, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13791743075293594, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10843148082494736, @@ -3417,6 +4093,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5252405539802958, + "pairwise_causal_calibration_ece": 0.13238341112315294, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.10074789459755354, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1992303878068924, @@ -3444,6 +4124,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5090186848560045, + "pairwise_causal_calibration_ece": 0.06240988657256685, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2836072551302167, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06855231523513794, @@ -3471,6 +4155,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.3101626245489895, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3498,6 +4186,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.508636849456031, + "pairwise_causal_calibration_ece": 0.29435113517031675, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2587694523429491, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14127866923809052, @@ -3525,6 +4217,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.52, + "pairwise_causal_calibration_confidence": 0.5138498006723948, + "pairwise_causal_calibration_ece": 0.006150199327605188, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.20665449797753987, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3552,6 +4248,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5111297696744155, + "pairwise_causal_calibration_ece": 0.01112976967441548, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2819397366030672, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09991621971130371, @@ -3579,6 +4279,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5107043594970093, + "pairwise_causal_calibration_ece": 0.02500992621727638, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.0578055103471221, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3606,6 +4310,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.516636818480975, + "pairwise_causal_calibration_ece": 0.07427227242811596, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.19671662459179742, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3633,6 +4341,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.6954287552895091, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3660,6 +4372,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8, + "pairwise_causal_calibration_confidence": 0.5098442555481146, + "pairwise_causal_calibration_ece": 0.2901557444518854, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.5588708482544975, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3687,6 +4403,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5056687364786325, + "pairwise_causal_calibration_ece": 0.21995445076434683, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.6378944690046493, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3714,6 +4434,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5078411572088926, + "pairwise_causal_calibration_ece": 0.13501598564825035, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.5738964308429505, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06987608969211578, @@ -3741,6 +4465,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7777777777777778, + "pairwise_causal_calibration_confidence": 0.5070230790958891, + "pairwise_causal_calibration_ece": 0.27075469868188873, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.6010746259173304, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3768,6 +4496,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5057799962955093, + "pairwise_causal_calibration_ece": 0.04977555926004629, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.6374969590449568, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3795,6 +4527,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5184867483888203, + "pairwise_causal_calibration_ece": 0.01848674838882025, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2871776156075902, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06764055788516998, @@ -3822,6 +4558,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5152073709393564, + "pairwise_causal_calibration_ece": 0.05622120048921497, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.24387011799339325, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19178248941898346, @@ -3849,6 +4589,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5198017998119995, + "pairwise_causal_calibration_ece": 0.16265894266914233, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2642625319749369, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1624142825603485, @@ -3876,6 +4620,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5130055389494003, + "pairwise_causal_calibration_ece": 0.04871982466368596, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.28149350829315467, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5673043727874756, @@ -3903,6 +4651,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5135673267124184, + "pairwise_causal_calibration_ece": 0.09357553043043865, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3504006480636836, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.8133031129837036, @@ -3930,6 +4682,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5094448174745808, + "pairwise_causal_calibration_ece": 0.2762694682397049, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.29289339637189027, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.216936856508255, @@ -3957,6 +4713,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5078543932512505, + "pairwise_causal_calibration_ece": 0.2578543932512505, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27542788251735256, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664863586425781, @@ -3984,6 +4744,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5174370315106026, + "pairwise_causal_calibration_ece": 0.2674370315106026, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26540267342818874, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18062961101531982, @@ -4011,6 +4775,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6153846153846154, + "pairwise_causal_calibration_confidence": 0.5096441739491412, + "pairwise_causal_calibration_ece": 0.10574044143547423, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.3320023318656702, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4038,6 +4806,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5063674605891497, + "pairwise_causal_calibration_ece": 0.18493888916057827, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.29579179447511267, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4065,6 +4837,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5121892620224119, + "pairwise_causal_calibration_ece": 0.012189262022411884, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3067677760628564, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4085,7 +4861,18 @@ ], "schema_version": 1, "selected_lambda": 0.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, "summary": { "base_success": { "macro_by_seed": { @@ -5391,6 +6178,646 @@ }, "n": 144 }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5192895132807416, + "low": 0.45512827409757234, + "mean": 0.4850536070492211, + "n": 114, + "num_samples": 1000 + }, + "mean": 0.4850536070492211, + "per_seed": { + "1073161778/train0": 0.5, + "1073161778/train1": 0.5714285714285714, + "1073161778/train2": 0.6111111111111112, + "1113371261/train0": 0.39285714285714285, + "1113371261/train1": 0.5714285714285714, + "1113371261/train2": 0.35714285714285715, + "1151410346/train0": 0.7857142857142857, + "1151410346/train1": 0.6428571428571429, + "1151410346/train2": 0.25, + "1261930456/train0": 0.2857142857142857, + "1261930456/train1": 0.6785714285714286, + "1261930456/train2": 0.6428571428571429, + "1324133122/train0": 0.42857142857142855, + "1324133122/train1": 0.5714285714285714, + "1324133122/train2": 0.25, + "1324642411/train0": 0.25, + "1324642411/train1": 0.5714285714285714, + "1324642411/train2": 0.8, + "1375492786/train1": 0.5, + "1375492786/train2": 0.35714285714285715, + "141984615/train0": 0.4642857142857143, + "141984615/train1": 0.6785714285714286, + "141984615/train2": 0.17857142857142858, + "1461069713/train0": 0.5, + "1461069713/train1": 0.5, + "1461069713/train2": 0.6785714285714286, + "1480856810/train0": 0.6666666666666666, + "1480856810/train1": 0.32142857142857145, + "1480856810/train2": 0.5555555555555556, + "1509980664/train0": 0.17857142857142858, + "1509980664/train1": 0.36, + "1509980664/train2": 0.6071428571428571, + "1522272686/train0": 0.42857142857142855, + "1522272686/train1": 0.75, + "1522272686/train2": 0.4642857142857143, + "1549324805/train0": 0.42857142857142855, + "1549324805/train1": 0.25, + "1549324805/train2": 0.5, + "1578990937/train0": 0.5714285714285714, + "1578990937/train1": 0.5357142857142857, + "1578990937/train2": 0.4642857142857143, + "1790188787/train0": 0.14285714285714285, + "1790188787/train2": 0.2857142857142857, + "1822210471/train0": 0.39285714285714285, + "1822210471/train1": 0.32142857142857145, + "1822210471/train2": 0.5357142857142857, + "1845489013/train0": 0.39285714285714285, + "1845489013/train1": 0.5714285714285714, + "1845489013/train2": 0.32142857142857145, + "1999022574/train0": 0.24, + "1999022574/train1": 0.4090909090909091, + "1999022574/train2": 0.5, + "2012211287/train0": 0.5357142857142857, + "2012211287/train1": 0.5714285714285714, + "2012211287/train2": 0.7272727272727273, + "2038594191/train0": 0.5185185185185185, + "2038594191/train1": 0.5714285714285714, + "2038594191/train2": 0.39285714285714285, + "2054899126/train0": 0.5714285714285714, + "2054899126/train1": 0.6785714285714286, + "2054899126/train2": 0.35714285714285715, + "2061635888/train0": 0.35714285714285715, + "2061635888/train1": 0.4642857142857143, + "2061635888/train2": 0.5, + "211245842/train0": 0.5714285714285714, + "211245842/train1": 0.7857142857142857, + "211245842/train2": 0.7857142857142857, + "278008524/train0": 0.4951298701298701, + "278008524/train1": 0.6785714285714286, + "278008524/train2": 0.7421428571428572, + "307068879/train0": 0.4642857142857143, + "307068879/train1": 0.32142857142857145, + "307068879/train2": 0.4642857142857143, + "412102563/train0": 0.14285714285714285, + "412102563/train1": 0.2857142857142857, + "412102563/train2": 0.32142857142857145, + "440577711/train0": 0.14285714285714285, + "440577711/train1": 1.0, + "441046872/train0": 0.2857142857142857, + "441046872/train1": 0.32142857142857145, + "441046872/train2": 0.14285714285714285, + "509430392/train0": 0.5357142857142857, + "509430392/train1": 0.9090909090909091, + "509430392/train2": 0.7777777777777778, + "585904130/train0": 0.5555555555555556, + "585904130/train1": 0.42857142857142855, + "585904130/train2": 0.5357142857142857, + "608016290/train0": 0.6428571428571429, + "608016290/train1": 0.3888888888888889, + "608016290/train2": 0.32142857142857145, + "626486813/train0": 0.4821428571428571, + "626486813/train1": 0.4642857142857143, + "626486813/train2": 0.5892857142857142, + "636680137/train0": 0.5, + "636680137/train1": 0.5357142857142857, + "636680137/train2": 0.6785714285714286, + "655884605/train0": 0.25, + "655884605/train1": 0.2857142857142857, + "655884605/train2": 0.5714285714285714, + "672328262/train0": 0.7272727272727273, + "672328262/train1": 0.6666666666666666, + "672328262/train2": 0.12, + "701667486/train0": 0.3076923076923077, + "701667486/train1": 0.14285714285714285, + "701667486/train2": 0.6153846153846154, + "747260437/train0": 0.4642857142857143, + "747260437/train1": 0.6428571428571429, + "747260437/train2": 0.21428571428571427, + "750448296/train0": 0.6331168831168831, + "750448296/train1": 0.5787545787545787, + "750448296/train2": 0.6017316017316018, + "767747424/train0": 0.6190476190476191, + "767747424/train1": 0.4642857142857143, + "767747424/train2": 0.6339285714285715 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5386506174864383, + "low": 0.461126146580692, + "mean": 0.501551201750659, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.501551201750659, + "per_task": { + "LiftPegUpright-v1": 0.4580086580086581, + "PickCube-v1": 0.5476190476190476, + "PullCube-v1": 0.527235771414876, + "PushCube-v1": 0.5353896103896104, + "StackCube-v1": 0.4395029213211031 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5291449125777485, + "low": 0.46962283432432667, + "mean": 0.4993105512508497, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.513199034267649, + "low": 0.5112501078754139, + "mean": 0.5121844772256707, + "n": 114, + "num_samples": 1000 + }, + "mean": 0.5121844772256707, + "per_seed": { + "1073161778/train0": 0.5098566341155361, + "1073161778/train1": 0.5091112114195717, + "1073161778/train2": 0.5089181730493006, + "1113371261/train0": 0.5198284636786039, + "1113371261/train1": 0.5196427235751272, + "1113371261/train2": 0.5198017998119995, + "1151410346/train0": 0.5145081193786424, + "1151410346/train1": 0.5069167821110796, + "1151410346/train2": 0.5078543932512505, + "1261930456/train0": 0.5097695136941484, + "1261930456/train1": 0.5085443341947974, + "1261930456/train2": 0.5078411572088926, + "1324133122/train0": 0.5224226018004667, + "1324133122/train1": 0.5227821807492782, + "1324133122/train2": 0.5174370315106026, + "1324642411/train0": 0.5088647318101668, + "1324642411/train1": 0.508953471583655, + "1324642411/train2": 0.5098442555481146, + "1375492786/train1": 0.5070611656440998, + "1375492786/train2": 0.5065725175893429, + "141984615/train0": 0.507901687873458, + "141984615/train1": 0.5164579122807452, + "141984615/train2": 0.5171743151724543, + "1461069713/train0": 0.5097775167745581, + "1461069713/train1": 0.5045038789520366, + "1461069713/train2": 0.5110427839399747, + "1480856810/train0": 0.5092043778610017, + "1480856810/train1": 0.509309423579836, + "1480856810/train2": 0.5057799962955093, + "1509980664/train0": 0.5107026203890623, + "1509980664/train1": 0.5100155711510672, + "1509980664/train2": 0.5135673267124184, + "1522272686/train0": 0.5250784055152783, + "1522272686/train1": 0.5105908420311568, + "1522272686/train2": 0.5231037401616476, + "1549324805/train0": 0.5082800251982494, + "1549324805/train1": 0.5088781157128205, + "1549324805/train2": 0.5111297696744155, + "1578990937/train0": 0.5158496642258744, + "1578990937/train1": 0.5115243772227345, + "1578990937/train2": 0.5130055389494003, + "1790188787/train0": 0.5040985707207823, + "1790188787/train2": 0.5056687364786325, + "1822210471/train0": 0.5148884431169273, + "1822210471/train1": 0.5177759880984708, + "1822210471/train2": 0.5107043594970093, + "1845489013/train0": 0.5108692392445339, + "1845489013/train1": 0.5114463604438318, + "1845489013/train2": 0.5089957841341125, + "1999022574/train0": 0.5186386660397809, + "1999022574/train1": 0.5119334766574462, + "1999022574/train2": 0.5121892620224119, + "2012211287/train0": 0.5079042501299456, + "2012211287/train1": 0.5038741047450654, + "2012211287/train2": 0.5094648157635078, + "2038594191/train0": 0.520866835944586, + "2038594191/train1": 0.5261612748851421, + "2038594191/train2": 0.5252405539802958, + "2054899126/train0": 0.5089660423581928, + "2054899126/train1": 0.5076812671130492, + "2054899126/train2": 0.5094631537502677, + "2061635888/train0": 0.5193863539393203, + "2061635888/train1": 0.5172736961846305, + "2061635888/train2": 0.5184867483888203, + "211245842/train0": 0.5093770540150734, + "211245842/train1": 0.5065274413865767, + "211245842/train2": 0.5094448174745808, + "278008524/train0": 0.507580765557335, + "278008524/train1": 0.5096338287511724, + "278008524/train2": 0.5138432222487193, + "307068879/train0": 0.517127112474433, + "307068879/train1": 0.5139732937786187, + "307068879/train2": 0.5182153571387064, + "412102563/train0": 0.5110298208425068, + "412102563/train1": 0.507869130131741, + "412102563/train2": 0.5063674605891497, + "440577711/train0": 0.5052959089864312, + "440577711/train1": 0.5089048821442496, + "441046872/train0": 0.5192681731031695, + "441046872/train1": 0.5155934510473184, + "441046872/train2": 0.5184906548624142, + "509430392/train0": 0.5059629842951442, + "509430392/train1": 0.5089098400968083, + "509430392/train2": 0.5070230790958891, + "585904130/train0": 0.5104367146945467, + "585904130/train1": 0.5082796231877765, + "585904130/train2": 0.5113970881858346, + "608016290/train0": 0.5093751939822863, + "608016290/train1": 0.5098943896149679, + "608016290/train2": 0.5120271305387815, + "626486813/train0": 0.5113876468965113, + "626486813/train1": 0.5116297285361195, + "626486813/train2": 0.514313264785462, + "636680137/train0": 0.5110307445144335, + "636680137/train1": 0.5087141999697643, + "636680137/train2": 0.5078943863682678, + "655884605/train0": 0.5196382790206474, + "655884605/train1": 0.5166281511611449, + "655884605/train2": 0.5152073709393564, + "672328262/train0": 0.5152932666576685, + "672328262/train1": 0.5278734108303779, + "672328262/train2": 0.5211353121802774, + "701667486/train0": 0.511181830700846, + "701667486/train1": 0.5114740860753181, + "701667486/train2": 0.5096441739491412, + "747260437/train0": 0.5082874271968991, + "747260437/train1": 0.5105870204506016, + "747260437/train2": 0.508636849456031, + "750448296/train0": 0.5114539295387354, + "750448296/train1": 0.513396376538758, + "750448296/train2": 0.5153965226212408, + "767747424/train0": 0.5090917071150636, + "767747424/train1": 0.5071819889949212, + "767747424/train2": 0.5087171759695033 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5168590506457146, + "low": 0.5098670341709353, + "mean": 0.5130135956897443, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5130135956897443, + "per_task": { + "LiftPegUpright-v1": 0.5122062630509082, + "PickCube-v1": 0.5195909959026942, + "PullCube-v1": 0.5116387196406088, + "PushCube-v1": 0.5077484539746286, + "StackCube-v1": 0.5138835458798818 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5128434786771694, + "low": 0.511149559217292, + "mean": 0.511968607664348, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.16168606314924241, + "low": 0.12124817784013511, + "mean": 0.1417859954013588, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.1417859954013588, + "per_seed": { + "1073161778/train0": 0.009856634115536056, + "1073161778/train1": 0.06231736000899968, + "1073161778/train2": 0.10219293806181051, + "1113371261/train0": 0.12697132082146106, + "1113371261/train1": 0.0517858478534442, + "1113371261/train2": 0.16265894266914233, + "1151410346/train0": 0.2712061663356433, + "1151410346/train1": 0.1359403607460633, + "1151410346/train2": 0.2578543932512505, + "1261930456/train0": 0.22405522797986266, + "1261930456/train1": 0.17002709437663122, + "1261930456/train2": 0.13501598564825035, + "1324133122/train0": 0.09385117322903819, + "1324133122/train1": 0.04864639067929322, + "1324133122/train2": 0.2674370315106026, + "1324642411/train0": 0.2588647318101668, + "1324642411/train1": 0.06247509984491639, + "1324642411/train2": 0.2901557444518854, + "1375492786/train0": 0.0, + "1375492786/train1": 0.007061165644099754, + "1375492786/train2": 0.1494296604464857, + "141984615/train0": 0.04361597358774372, + "141984615/train1": 0.16211351629068338, + "141984615/train2": 0.33860288660102567, + "1461069713/train0": 0.00977751677455807, + "1461069713/train1": 0.004503878952036633, + "1461069713/train2": 0.1675286446314539, + "1480856810/train0": 0.15746228880566493, + "1480856810/train1": 0.18788085215126454, + "1480856810/train2": 0.04977555926004629, + "1509980664/train0": 0.33213119181763373, + "1509980664/train1": 0.15001557115106723, + "1509980664/train2": 0.09357553043043865, + "1522272686/train0": 0.09650697694384974, + "1522272686/train1": 0.2394091579688432, + "1522272686/train2": 0.05881802587593332, + "1549324805/train0": 0.0797085966268209, + "1549324805/train1": 0.2588781157128205, + "1549324805/train2": 0.01112976967441548, + "1578990937/train0": 0.05557890720269698, + "1578990937/train1": 0.024189908491551226, + "1578990937/train2": 0.04871982466368596, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.3612414278636395, + "1790188787/train1": 0.0, + "1790188787/train2": 0.21995445076434683, + "1822210471/train0": 0.12203130025978443, + "1822210471/train1": 0.19634741666989936, + "1822210471/train2": 0.02500992621727638, + "1845489013/train0": 0.11801209638739102, + "1845489013/train1": 0.05998221098473955, + "1845489013/train2": 0.18756721270554105, + "1999022574/train0": 0.27863866603978094, + "1999022574/train1": 0.10284256756653704, + "1999022574/train2": 0.012189262022411884, + "2012211287/train0": 0.027810035584340143, + "2012211287/train1": 0.06755446668350595, + "2012211287/train2": 0.21780791150921952, + "2038594191/train0": 0.0023483174260675588, + "2038594191/train1": 0.045267296543429314, + "2038594191/train2": 0.13238341112315294, + "2054899126/train0": 0.031231264535189318, + "2054899126/train1": 0.08544508072918972, + "2054899126/train2": 0.0761601483037053, + "2061635888/train0": 0.16224349679646316, + "2061635888/train1": 0.05298798189891618, + "2061635888/train2": 0.01848674838882025, + "211245842/train0": 0.062051517413498014, + "211245842/train1": 0.279186844327709, + "211245842/train2": 0.2762694682397049, + "278008524/train0": 0.03747258161333558, + "278008524/train1": 0.16893759982025613, + "278008524/train2": 0.22829963489413796, + "307068879/train0": 0.05284139818871869, + "307068879/train1": 0.19254472235004721, + "307068879/train2": 0.053929642852992066, + "412102563/train0": 0.36817267798536396, + "412102563/train1": 0.22215484441745526, + "412102563/train2": 0.18493888916057827, + "440577711/train0": 0.3624387661292883, + "440577711/train1": 0.49109511785575044, + "440577711/train2": 0.0, + "441046872/train0": 0.23355388738888383, + "441046872/train1": 0.1941648796187469, + "441046872/train2": 0.37563351200527134, + "509430392/train0": 0.029751301419141507, + "509430392/train1": 0.4001810689941008, + "509430392/train2": 0.27075469868188873, + "585904130/train0": 0.045118840861008924, + "585904130/train1": 0.07970819461634798, + "585904130/train2": 0.024317197528451073, + "608016290/train0": 0.13348194887485665, + "608016290/train1": 0.12100550072607902, + "608016290/train2": 0.19059855911021, + "626486813/train0": 0.04992710785321339, + "626486813/train1": 0.14403731951037171, + "626486813/train2": 0.07497244950025217, + "636680137/train0": 0.011030744514433533, + "636680137/train1": 0.027000085744521396, + "636680137/train2": 0.17067704220316082, + "655884605/train0": 0.2696382790206474, + "655884605/train1": 0.2309138654468592, + "655884605/train2": 0.05622120048921497, + "672328262/train0": 0.2119794606150588, + "672328262/train1": 0.13879325583628876, + "672328262/train2": 0.4011353121802774, + "701667486/train0": 0.2034895230085383, + "701667486/train1": 0.36861694321817523, + "701667486/train2": 0.10574044143547423, + "747260437/train0": 0.04400171291118482, + "747260437/train1": 0.13227012240654135, + "747260437/train2": 0.29435113517031675, + "750448296/train0": 0.21751195260366055, + "750448296/train1": 0.20120295855494272, + "750448296/train2": 0.08633507911036098, + "767747424/train0": 0.10703103606299277, + "767747424/train1": 0.10997339402687645, + "767747424/train2": 0.21759707156370559 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1793142132847833, + "low": 0.1264255566997398, + "mean": 0.15014958104027704, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.15014958104027704, + "per_task": { + "LiftPegUpright-v1": 0.13568622647194786, + "PickCube-v1": 0.13157805359620875, + "PullCube-v1": 0.11664272491716678, + "PushCube-v1": 0.20395163422426912, + "StackCube-v1": 0.16288926599179274 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.16052506774717407, + "low": 0.12171972152997879, + "mean": 0.1404493550029261, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 25.141666666666666, + "low": 22.325, + "mean": 23.8625, + "n": 120, + "num_samples": 1000 + }, + "mean": 23.8625, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 18.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 28.0, + "1324642411/train2": 25.0, + "1375492786/train0": 0.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 18.0, + "1480856810/train1": 28.0, + "1480856810/train2": 27.0, + "1509980664/train0": 28.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 7.0, + "1790188787/train1": 0.0, + "1790188787/train2": 7.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 25.0, + "1999022574/train1": 22.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 28.0, + "2012211287/train2": 22.0, + "2038594191/train0": 27.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 25.0, + "278008524/train1": 28.0, + "278008524/train2": 26.5, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 7.0, + "412102563/train1": 7.0, + "412102563/train2": 28.0, + "440577711/train0": 7.0, + "440577711/train1": 7.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 28.0, + "509430392/train1": 22.0, + "509430392/train2": 27.0, + "585904130/train0": 18.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 18.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 22.0, + "672328262/train1": 18.0, + "672328262/train2": 25.0, + "701667486/train0": 13.0, + "701667486/train1": 7.0, + "701667486/train2": 13.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 26.0, + "750448296/train1": 23.0, + "750448296/train2": 26.0, + "767747424/train0": 21.0, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 26.880000000000003, + "low": 21.513333333333335, + "mean": 24.454747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 24.454747474747474, + "per_task": { + "LiftPegUpright-v1": 26.266666666666666, + "PickCube-v1": 28.0, + "PullCube-v1": 24.133333333333333, + "PushCube-v1": 19.055555555555557, + "StackCube-v1": 24.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 25.27777777777778, + "low": 22.53472222222222, + "mean": 23.958333333333332, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, "predicted_margin": { "macro_by_seed": { "ci": { @@ -7513,6 +8940,7 @@ }, "target": "success", "tau": 0.3152506249621047, + "threshold_scope": "global", "weights": [ 0.2317708333333339, -0.026217393987237325, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_seed.json index e0276e23112c0a141a468c630ef0c8f618d40fff..64af0d6b3359aade52e9e3049bb60d56c91aa264 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_seed.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_seed.json @@ -8,6 +8,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5098566341155361, + "pairwise_causal_calibration_ece": 0.009856634115536056, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.197805317803883, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -31,6 +35,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5091112114195717, + "pairwise_causal_calibration_ece": 0.06231736000899968, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1847762400277908, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7075161337852478, @@ -54,6 +62,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6111111111111112, + "pairwise_causal_calibration_confidence": 0.5089181730493006, + "pairwise_causal_calibration_ece": 0.10219293806181051, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.14402410971307644, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -77,6 +89,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5198284636786039, + "pairwise_causal_calibration_ece": 0.12697132082146106, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.246546487439865, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09061422944068909, @@ -100,6 +116,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5196427235751272, + "pairwise_causal_calibration_ece": 0.0517858478534442, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23873008123815037, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14092671871185303, @@ -123,6 +143,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5198017998119995, + "pairwise_causal_calibration_ece": 0.16265894266914233, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2642625319749369, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1624142825603485, @@ -146,6 +170,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5145081193786424, + "pairwise_causal_calibration_ece": 0.2712061663356433, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2766048493279699, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20478898286819458, @@ -169,6 +197,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5069167821110796, + "pairwise_causal_calibration_ece": 0.1359403607460633, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23010369929396896, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2086353600025177, @@ -192,6 +224,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5078543932512505, + "pairwise_causal_calibration_ece": 0.2578543932512505, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27542788251735256, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664863586425781, @@ -215,6 +251,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5097695136941484, + "pairwise_causal_calibration_ece": 0.22405522797986266, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6193304427041512, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06612107157707214, @@ -238,6 +278,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5085443341947974, + "pairwise_causal_calibration_ece": 0.17002709437663122, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.667791884975374, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05957312881946564, @@ -261,6 +305,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5078411572088926, + "pairwise_causal_calibration_ece": 0.13501598564825035, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.5738964308429505, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06987608969211578, @@ -284,6 +332,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5224226018004667, + "pairwise_causal_calibration_ece": 0.09385117322903819, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26820123426100245, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.056939899921417236, @@ -307,6 +359,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5227821807492782, + "pairwise_causal_calibration_ece": 0.04864639067929322, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.30466427900455206, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12235099077224731, @@ -330,6 +386,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5174370315106026, + "pairwise_causal_calibration_ece": 0.2674370315106026, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26540267342818874, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18062961101531982, @@ -353,6 +413,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5088647318101668, + "pairwise_causal_calibration_ece": 0.2588647318101668, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6712258551783686, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.17992261052131653, @@ -376,6 +440,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.508953471583655, + "pairwise_causal_calibration_ece": 0.06247509984491639, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7232270300064332, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1777031123638153, @@ -399,6 +467,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8, + "pairwise_causal_calibration_confidence": 0.5098442555481146, + "pairwise_causal_calibration_ece": 0.2901557444518854, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.5588708482544975, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -422,6 +494,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.1142441575920736, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -445,6 +519,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5070611656440998, + "pairwise_causal_calibration_ece": 0.007061165644099754, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1658630525898114, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09690030664205551, @@ -468,6 +546,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5065725175893429, + "pairwise_causal_calibration_ece": 0.1494296604464857, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13791743075293594, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10843148082494736, @@ -491,6 +573,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.507901687873458, + "pairwise_causal_calibration_ece": 0.04361597358774372, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20959728035122172, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.008802552707493305, @@ -514,6 +600,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5164579122807452, + "pairwise_causal_calibration_ece": 0.16211351629068338, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21183340573001233, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03755499795079231, @@ -537,6 +627,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5171743151724543, + "pairwise_causal_calibration_ece": 0.33860288660102567, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18308305510200482, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03313249349594116, @@ -560,6 +654,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5097775167745581, + "pairwise_causal_calibration_ece": 0.00977751677455807, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1969950204810189, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06010149046778679, @@ -583,6 +681,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5045038789520366, + "pairwise_causal_calibration_ece": 0.004503878952036633, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20397936444755985, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.030005255714058876, @@ -606,6 +708,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5110427839399747, + "pairwise_causal_calibration_ece": 0.1675286446314539, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27422519801828676, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09976043552160263, @@ -629,6 +735,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5092043778610017, + "pairwise_causal_calibration_ece": 0.15746228880566493, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.6561771188937511, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -652,6 +762,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.509309423579836, + "pairwise_causal_calibration_ece": 0.18788085215126454, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7595411023961984, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19969357550144196, @@ -675,6 +789,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5057799962955093, + "pairwise_causal_calibration_ece": 0.04977555926004629, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.6374969590449568, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -698,6 +816,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5107026203890623, + "pairwise_causal_calibration_ece": 0.33213119181763373, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2595476416603391, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -721,6 +843,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36, + "pairwise_causal_calibration_confidence": 0.5100155711510672, + "pairwise_causal_calibration_ece": 0.15001557115106723, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.276906017761115, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -744,6 +870,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5135673267124184, + "pairwise_causal_calibration_ece": 0.09357553043043865, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3504006480636836, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.8133031129837036, @@ -767,6 +897,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5250784055152783, + "pairwise_causal_calibration_ece": 0.09650697694384974, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3284611884666808, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16821590065956116, @@ -790,6 +924,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5105908420311568, + "pairwise_causal_calibration_ece": 0.2394091579688432, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2760379294226463, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07200749218463898, @@ -813,6 +951,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5231037401616476, + "pairwise_causal_calibration_ece": 0.05881802587593332, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3311257655653472, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15220001339912415, @@ -836,6 +978,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082800251982494, + "pairwise_causal_calibration_ece": 0.0797085966268209, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23866768065123248, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04792773723602295, @@ -859,6 +1005,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5088781157128205, + "pairwise_causal_calibration_ece": 0.2588781157128205, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3111189505567141, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2170039862394333, @@ -882,6 +1032,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5111297696744155, + "pairwise_causal_calibration_ece": 0.01112976967441548, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2819397366030672, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09991621971130371, @@ -905,6 +1059,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5158496642258744, + "pairwise_causal_calibration_ece": 0.05557890720269698, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26276318940133603, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20236456394195557, @@ -928,6 +1086,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5115243772227345, + "pairwise_causal_calibration_ece": 0.024189908491551226, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2479399896937514, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20639251172542572, @@ -951,6 +1113,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5130055389494003, + "pairwise_causal_calibration_ece": 0.04871982466368596, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.28149350829315467, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5673043727874756, @@ -974,6 +1140,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.29301731712998935, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -997,6 +1165,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.3803269800393895, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1020,6 +1190,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.3463013216157089, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1043,6 +1215,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5040985707207823, + "pairwise_causal_calibration_ece": 0.3612414278636395, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.6291681545836397, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1066,6 +1242,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.6121226617843838, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1089,6 +1267,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5056687364786325, + "pairwise_causal_calibration_ece": 0.21995445076434683, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.6378944690046493, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1112,6 +1294,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5148884431169273, + "pairwise_causal_calibration_ece": 0.12203130025978443, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11067121611907713, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1135,6 +1321,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5177759880984708, + "pairwise_causal_calibration_ece": 0.19634741666989936, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09669884825042896, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13508938252925873, @@ -1158,6 +1348,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5107043594970093, + "pairwise_causal_calibration_ece": 0.02500992621727638, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.0578055103471221, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1181,6 +1375,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5108692392445339, + "pairwise_causal_calibration_ece": 0.11801209638739102, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14108783056103189, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02055191993713379, @@ -1204,6 +1402,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5114463604438318, + "pairwise_causal_calibration_ece": 0.05998221098473955, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1226666983652919, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05916740372776985, @@ -1227,6 +1429,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5089957841341125, + "pairwise_causal_calibration_ece": 0.18756721270554105, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07991407615376411, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1250,6 +1456,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.24, + "pairwise_causal_calibration_confidence": 0.5186386660397809, + "pairwise_causal_calibration_ece": 0.27863866603978094, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.25444049345463426, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1273,6 +1483,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4090909090909091, + "pairwise_causal_calibration_confidence": 0.5119334766574462, + "pairwise_causal_calibration_ece": 0.10284256756653704, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.24790697988984683, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1296,6 +1510,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5121892620224119, + "pairwise_causal_calibration_ece": 0.012189262022411884, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3067677760628564, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1319,6 +1537,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5079042501299456, + "pairwise_causal_calibration_ece": 0.027810035584340143, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20092600070845926, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1342,6 +1564,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5038741047450654, + "pairwise_causal_calibration_ece": 0.06755446668350595, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16349292872459203, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1365,6 +1591,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5094648157635078, + "pairwise_causal_calibration_ece": 0.21780791150921952, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.17995610986243743, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1388,6 +1618,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5185185185185185, + "pairwise_causal_calibration_confidence": 0.520866835944586, + "pairwise_causal_calibration_ece": 0.0023483174260675588, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.11438425065440704, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1411,6 +1645,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5261612748851421, + "pairwise_causal_calibration_ece": 0.045267296543429314, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17159672738556292, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15240873396396637, @@ -1434,6 +1672,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5252405539802958, + "pairwise_causal_calibration_ece": 0.13238341112315294, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.10074789459755354, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1992303878068924, @@ -1457,6 +1699,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5089660423581928, + "pairwise_causal_calibration_ece": 0.031231264535189318, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.275573013269513, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0780495330691338, @@ -1480,6 +1726,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5076812671130492, + "pairwise_causal_calibration_ece": 0.08544508072918972, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.2841151465083533, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0777441188693047, @@ -1503,6 +1753,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5094631537502677, + "pairwise_causal_calibration_ece": 0.0761601483037053, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.2679532638886026, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.053164016455412, @@ -1526,6 +1780,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5193863539393203, + "pairwise_causal_calibration_ece": 0.16224349679646316, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20663971147287907, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.0642656534910202, @@ -1549,6 +1807,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5172736961846305, + "pairwise_causal_calibration_ece": 0.05298798189891618, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.26282861689601783, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07914192974567413, @@ -1572,6 +1834,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5184867483888203, + "pairwise_causal_calibration_ece": 0.01848674838882025, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2871776156075902, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06764055788516998, @@ -1595,6 +1861,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5093770540150734, + "pairwise_causal_calibration_ece": 0.062051517413498014, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2818040383848624, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1618,6 +1888,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5065274413865767, + "pairwise_causal_calibration_ece": 0.279186844327709, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27950117722622586, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22683429718017578, @@ -1641,6 +1915,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5094448174745808, + "pairwise_causal_calibration_ece": 0.2762694682397049, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.29289339637189027, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.216936856508255, @@ -1664,6 +1942,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4951298701298701, + "pairwise_causal_calibration_confidence": 0.507580765557335, + "pairwise_causal_calibration_ece": 0.03747258161333558, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.10924904877533874, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0352020561695099, @@ -1687,6 +1969,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5096338287511724, + "pairwise_causal_calibration_ece": 0.16893759982025613, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12740485358567616, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0799707099795341, @@ -1710,6 +1996,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7421428571428572, + "pairwise_causal_calibration_confidence": 0.5138432222487193, + "pairwise_causal_calibration_ece": 0.22829963489413796, + "pairwise_causal_calibration_pairs": 26.5, "predicted_margin": 0.18675708897764087, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.1454492062330246, @@ -1733,6 +2023,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.517127112474433, + "pairwise_causal_calibration_ece": 0.05284139818871869, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21756370458471622, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20391923189163208, @@ -1756,6 +2050,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5139732937786187, + "pairwise_causal_calibration_ece": 0.19254472235004721, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1801716769818562, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20632563531398773, @@ -1779,6 +2077,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5182153571387064, + "pairwise_causal_calibration_ece": 0.053929642852992066, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20469744001592355, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20428544282913208, @@ -1802,6 +2104,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5110298208425068, + "pairwise_causal_calibration_ece": 0.36817267798536396, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.2782043046573905, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1825,6 +2131,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.507869130131741, + "pairwise_causal_calibration_ece": 0.22215484441745526, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.2854092085136363, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1848,6 +2158,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5063674605891497, + "pairwise_causal_calibration_ece": 0.18493888916057827, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.29579179447511267, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1871,6 +2185,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5052959089864312, + "pairwise_causal_calibration_ece": 0.3624387661292883, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.6870668145379031, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1894,6 +2212,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 1.0, + "pairwise_causal_calibration_confidence": 0.5089048821442496, + "pairwise_causal_calibration_ece": 0.49109511785575044, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7609842811296632, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1917,6 +2239,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.6954287552895091, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1940,6 +2264,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5192681731031695, + "pairwise_causal_calibration_ece": 0.23355388738888383, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2043317342807088, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5837622880935669, @@ -1963,6 +2291,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5155934510473184, + "pairwise_causal_calibration_ece": 0.1941648796187469, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16033886964133004, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7209141850471497, @@ -1986,6 +2318,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5184906548624142, + "pairwise_causal_calibration_ece": 0.37563351200527134, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19688713080112988, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.21979393064975739, @@ -2009,6 +2345,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5059629842951442, + "pairwise_causal_calibration_ece": 0.029751301419141507, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6471664922001317, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12932512164115906, @@ -2032,6 +2372,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9090909090909091, + "pairwise_causal_calibration_confidence": 0.5089098400968083, + "pairwise_causal_calibration_ece": 0.4001810689941008, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.6675461655230028, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2055,6 +2399,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7777777777777778, + "pairwise_causal_calibration_confidence": 0.5070230790958891, + "pairwise_causal_calibration_ece": 0.27075469868188873, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.6010746259173304, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2078,6 +2426,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5104367146945467, + "pairwise_causal_calibration_ece": 0.045118840861008924, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.09578732958371006, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2101,6 +2453,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082796231877765, + "pairwise_causal_calibration_ece": 0.07970819461634798, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.10249988539655566, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1290181279182434, @@ -2124,6 +2480,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5113970881858346, + "pairwise_causal_calibration_ece": 0.024317197528451073, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09962893251516124, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2147,6 +2507,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5093751939822863, + "pairwise_causal_calibration_ece": 0.13348194887485665, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1717547088237561, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2170,6 +2534,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3888888888888889, + "pairwise_causal_calibration_confidence": 0.5098943896149679, + "pairwise_causal_calibration_ece": 0.12100550072607902, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.20877294341834293, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2193,6 +2561,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5120271305387815, + "pairwise_causal_calibration_ece": 0.19059855911021, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13802477297102525, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664259791374207, @@ -2216,6 +2588,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4821428571428571, + "pairwise_causal_calibration_confidence": 0.5113876468965113, + "pairwise_causal_calibration_ece": 0.04992710785321339, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15020438296667366, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04293955350294709, @@ -2239,6 +2615,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5116297285361195, + "pairwise_causal_calibration_ece": 0.14403731951037171, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17852089799806223, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02655014442279935, @@ -2262,6 +2642,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5892857142857142, + "pairwise_causal_calibration_confidence": 0.514313264785462, + "pairwise_causal_calibration_ece": 0.07497244950025217, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14307776706871508, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03389316890388727, @@ -2285,6 +2669,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5110307445144335, + "pairwise_causal_calibration_ece": 0.011030744514433533, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11748877632482482, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09099660068750381, @@ -2308,6 +2696,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5087141999697643, + "pairwise_causal_calibration_ece": 0.027000085744521396, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1312853076823468, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08514270931482315, @@ -2331,6 +2723,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5078943863682678, + "pairwise_causal_calibration_ece": 0.17067704220316082, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1345855010442043, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08911379426717758, @@ -2354,6 +2750,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5196382790206474, + "pairwise_causal_calibration_ece": 0.2696382790206474, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.24752873446742565, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1631268858909607, @@ -2377,6 +2777,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5166281511611449, + "pairwise_causal_calibration_ece": 0.2309138654468592, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23791187673632117, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15427637100219727, @@ -2400,6 +2804,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5152073709393564, + "pairwise_causal_calibration_ece": 0.05622120048921497, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.24387011799339325, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19178248941898346, @@ -2423,6 +2831,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5152932666576685, + "pairwise_causal_calibration_ece": 0.2119794606150588, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.0886735921808986, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2446,6 +2858,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5278734108303779, + "pairwise_causal_calibration_ece": 0.13879325583628876, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.18825713114760673, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2469,6 +2885,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.12, + "pairwise_causal_calibration_confidence": 0.5211353121802774, + "pairwise_causal_calibration_ece": 0.4011353121802774, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.08486096062823595, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2492,6 +2912,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.511181830700846, + "pairwise_causal_calibration_ece": 0.2034895230085383, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.2713698658138815, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2515,6 +2939,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5114740860753181, + "pairwise_causal_calibration_ece": 0.36861694321817523, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.30277388356719176, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2538,6 +2966,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6153846153846154, + "pairwise_causal_calibration_confidence": 0.5096441739491412, + "pairwise_causal_calibration_ece": 0.10574044143547423, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.3320023318656702, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2561,6 +2993,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5082874271968991, + "pairwise_causal_calibration_ece": 0.04400171291118482, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.22414376708358918, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13453228771686554, @@ -2584,6 +3020,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5105870204506016, + "pairwise_causal_calibration_ece": 0.13227012240654135, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27086090181173816, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06424945592880249, @@ -2607,6 +3047,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.508636849456031, + "pairwise_causal_calibration_ece": 0.29435113517031675, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2587694523429491, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14127866923809052, @@ -2630,6 +3074,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.6331168831168831, + "pairwise_causal_calibration_confidence": 0.5114539295387354, + "pairwise_causal_calibration_ece": 0.21751195260366055, + "pairwise_causal_calibration_pairs": 26.0, "predicted_margin": 0.16432581896894002, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7655241700510184, @@ -2653,6 +3101,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5787545787545787, + "pairwise_causal_calibration_confidence": 0.513396376538758, + "pairwise_causal_calibration_ece": 0.20120295855494272, + "pairwise_causal_calibration_pairs": 23.0, "predicted_margin": 0.1977895366681389, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7756740699211756, @@ -2676,6 +3128,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.6017316017316018, + "pairwise_causal_calibration_confidence": 0.5153965226212408, + "pairwise_causal_calibration_ece": 0.08633507911036098, + "pairwise_causal_calibration_pairs": 26.0, "predicted_margin": 0.2141060722436662, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7652813941240311, @@ -2699,6 +3155,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.6190476190476191, + "pairwise_causal_calibration_confidence": 0.5090917071150636, + "pairwise_causal_calibration_ece": 0.10703103606299277, + "pairwise_causal_calibration_pairs": 21.0, "predicted_margin": 0.203919201209952, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0471355519257486, @@ -2722,6 +3182,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5071819889949212, + "pairwise_causal_calibration_ece": 0.10997339402687645, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20002804565583177, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10832798760384321, @@ -2745,6 +3209,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.6339285714285715, + "pairwise_causal_calibration_confidence": 0.5087171759695033, + "pairwise_causal_calibration_ece": 0.21759707156370559, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19413727025774286, "proposal_oracle_success": 0.25, "proposal_oracle_utility": 0.5676866680150852, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_task.json index 79b646b69d45597e2a91b73885e2a0f5e7d67653..f03379ad2b24b8ff7d2b165488def725cd01265e 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_task.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/metrics_by_task.json @@ -8,6 +8,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.13333333333333333, + "pairwise_causal_calibration_accuracy": 0.4580086580086581, + "pairwise_causal_calibration_confidence": 0.5122062630509082, + "pairwise_causal_calibration_ece": 0.13568622647194786, + "pairwise_causal_calibration_pairs": 26.266666666666666, "predicted_margin": 0.18356824589060186, "proposal_oracle_success": 0.4666666666666667, "proposal_oracle_utility": 1.1608628551165263, @@ -31,6 +35,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5476190476190476, + "pairwise_causal_calibration_confidence": 0.5195909959026942, + "pairwise_causal_calibration_ece": 0.13157805359620875, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3118749611515581, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1308078020811081, @@ -54,6 +62,10 @@ "hidden_chart_oracle_success": 0.52, "hidden_chart_oracle_utility": 1.1467878860235214, "outcome_ptr": 0.38666666666666666, + "pairwise_causal_calibration_accuracy": 0.527235771414876, + "pairwise_causal_calibration_confidence": 0.5116387196406088, + "pairwise_causal_calibration_ece": 0.11664272491716678, + "pairwise_causal_calibration_pairs": 24.133333333333333, "predicted_margin": 0.18544317093400783, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7346437490296861, @@ -77,6 +89,10 @@ "hidden_chart_oracle_success": 0.8333333333333334, "hidden_chart_oracle_utility": 1.7015800600250561, "outcome_ptr": 0.8333333333333334, + "pairwise_causal_calibration_accuracy": 0.5353896103896104, + "pairwise_causal_calibration_confidence": 0.5077484539746286, + "pairwise_causal_calibration_ece": 0.20395163422426912, + "pairwise_causal_calibration_pairs": 19.055555555555557, "predicted_margin": 0.6558894495703831, "proposal_oracle_success": 0.6111111111111112, "proposal_oracle_utility": 1.2712341505620215, @@ -100,6 +116,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.42424242424242425, + "pairwise_causal_calibration_accuracy": 0.4395029213211031, + "pairwise_causal_calibration_confidence": 0.5138835458798818, + "pairwise_causal_calibration_ece": 0.16288926599179274, + "pairwise_causal_calibration_pairs": 24.818181818181817, "predicted_margin": 0.2716308071762482, "proposal_oracle_success": 0.36363636363636365, "proposal_oracle_utility": 0.8693077586817018, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/table.tex index 393dbe81618c0750e5c83e4f36fdf2d5db14e10d..9d1f8dd24520df4b056f9ed50f9c4a5be7795ad4 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/table.tex +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/table.tex @@ -1,8 +1,8 @@ % Auto-generated by scripts/eval_learned_dominance_selector.py -\begin{tabular}{lrrrrrrrr} +\begin{tabular}{lrrrrrrrrr} \toprule -Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ \midrule -144 & 0.1806 & 0.8194 & 0.3333 & 0.3819 & 0.3819 & 0.4167 & 0.3681 & 0.1250 \\ +144 & 0.1806 & 0.8194 & 0.3333 & 0.3819 & 0.3819 & 0.4167 & 0.3681 & 0.1250 & 0.0055 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/train.log index 65ae422f8b9c6db47758322bb85b32bb0fe6229a..d0ef76364dff8a235ab92910ad28598733d29009 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/train.log +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_success_tanh_train_to_test/train.log @@ -1,4 +1,5 @@ trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise selected_lambda=0.0 tau=0.315251 calibration_input=runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..325cc06de704df623891a4340dbaf11804b76c28 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test --feature-set context_tangent --target utility_margin --k 8 --bootstrap-samples 1000 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c5f0199b92b0d833db59904ffb7e541cc87e7f6 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: context_tangent +fit_objective: pointwise +k: 8 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: None +success_bonus: 1.0 +target: utility_margin +threshold_scope: global diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d91fbacd145085f2d9a143f07cf1791ff492f9fc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "130d86b8c5ba5dc901ee48835454693bf42c361803c01f6b2d80590cb2493491", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "88d4bfe8c95f8d1579cbe59096cf665dc5f2dfdd99481a386a25e1f3cfa2b09d", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..58e60527653c049a1edc75cf9002db1286feb6ae --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0a7944fd2103dc224ae45d7ba8e291a9465bfc78 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics.json @@ -0,0 +1,9155 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.5416666666666666, + "fallback_rate": 0.4583333333333333, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3194444444444444, + "selected_success_gain_over_base": 0.020833333333333332, + "selected_utility": 0.7649397741333814, + "selector_regret": 0.051482520614323825, + "success_selector_gap": 0.024305555555555556, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_summary": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.5416666666666666, + "fallback_rate": 0.4583333333333333, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "pairwise_causal_calibration_accuracy": 0.5406537007023231, + "pairwise_causal_calibration_confidence": 0.5368345782053449, + "pairwise_causal_calibration_ece": 0.009091998384416941, + "pairwise_causal_calibration_pairs": 7404.0, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3194444444444444, + "selected_success_gain_over_base": 0.020833333333333332, + "selected_utility": 0.7649397741333814, + "selector_regret": 0.051482520614323825, + "success_selector_gap": 0.024305555555555556, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.3333333333333333, + "base_utility": 0.8338337515791258, + "coverage": 0.6111111111111112, + "fallback_rate": 0.3888888888888889, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.3611111111111111, + "pairwise_causal_calibration_accuracy": 0.5289459311851448, + "pairwise_causal_calibration_confidence": 0.5421422103946226, + "pairwise_causal_calibration_ece": 0.01449932655549304, + "pairwise_causal_calibration_pairs": 3662.0, + "proposal_oracle_success": 0.2222222222222222, + "proposal_oracle_utility": 0.6232009150553495, + "selected_success": 0.2986111111111111, + "selected_success_gain_over_base": -0.034722222222222224, + "selected_utility": 0.7566216745375035, + "selector_regret": 0.12803097097073785, + "success_selector_gap": 0.06944444444444445, + "success_support_gap": 0.5138888888888888, + "support_gap": 0.9091424554741631 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.182546837171281, + 1.8836221616495297, + -1.0601087908747155e-18, + 3.5, + 0.125, + 1.2092013888888888, + 0.7583173435164257, + 1.9059527259604592, + 8.0, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.36258169934639883, + -0.4588235294117674, + 0.1346405228758154, + 0.3127450980392166, + -0.5231481481481283, + -0.5494008714596817, + 0.5098039215686088, + -0.08622004357298363, + 1.0, + 0.2707248263888889, + 0.2306857638888889, + 0.15460957213589507, + -0.002221525274055441, + 0.4404452632491787, + -0.0732957456914139, + 0.1185007049029486, + 0.28929648765349864, + 0.37639624912602204, + 0.289247130454492, + 0.08760700280981837, + 0.021523393619443394, + 0.0013782045918762581, + 0.07315826858070472, + -0.36818566475348136, + 0.08511420341286187, + 0.10109417613744477, + 0.03422285977952318, + 0.07723466354137701, + 0.04658102504779688, + 0.17487594802838025, + -0.1345592589544443, + 0.07206917453893563, + 0.4632009652947697, + 0.13900196984549337, + 0.6286754180263314, + 0.4662843597309095, + 0.7146745238246189, + 1.1377109955808717, + 0.7123874865420576, + 0.42250050161641817, + 0.18250456237243876, + 0.34622435484521297, + 0.4512131481436275, + 0.6555436916565264, + 0.8817456337920804, + 0.6499884038219332, + 0.35018353749344694, + 0.1856164064041271, + 0.4309610689589236, + 0.3701181374425586, + 0.5064969249263717, + 0.9176829091027483, + 0.5763792715798546 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "target_task_hash_00", + "target_task_hash_01", + "target_task_hash_02", + "target_task_hash_03", + "target_task_hash_04", + "target_task_hash_05", + "target_task_hash_06", + "target_task_hash_07", + "source_task_hash_00", + "source_task_hash_01", + "source_task_hash_02", + "source_task_hash_03", + "source_task_hash_04", + "source_task_hash_05", + "source_task_hash_06", + "source_task_hash_07", + "instruction_hash_00", + "instruction_hash_01", + "instruction_hash_02", + "instruction_hash_03", + "instruction_hash_04", + "instruction_hash_05", + "instruction_hash_06", + "instruction_hash_07", + "source_target_same_task", + "instruction_chars_per_128", + "instruction_words_per_32", + "tangent_00", + "tangent_01", + "tangent_02", + "tangent_03", + "tangent_04", + "tangent_05", + "tangent_06", + "tangent_07", + "tangent_08", + "tangent_09", + "tangent_10", + "tangent_11", + "tangent_12", + "tangent_13", + "tangent_14", + "tangent_15", + "tangent_16", + "tangent_17", + "tangent_18", + "tangent_19", + "tangent_20", + "abs_tangent_00", + "abs_tangent_01", + "abs_tangent_02", + "abs_tangent_03", + "abs_tangent_04", + "abs_tangent_05", + "abs_tangent_06", + "abs_tangent_07", + "abs_tangent_08", + "abs_tangent_09", + "abs_tangent_10", + "abs_tangent_11", + "abs_tangent_12", + "abs_tangent_13", + "abs_tangent_14", + "abs_tangent_15", + "abs_tangent_16", + "abs_tangent_17", + "abs_tangent_18", + "abs_tangent_19", + "abs_tangent_20" + ], + "feature_set": "context_tangent", + "feature_std": [ + 1.0, + 2.4903288136152946, + 2.1478743840800227, + 0.999998700632758, + 2.29128884747792, + 0.3307199138830738, + 1.490882562848811, + 0.3689689239790769, + 0.7063143432867478, + 1e-06, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37487873363847424, + 0.3443939715604908, + 0.32513795823134695, + 0.3102098787644046, + 0.4154730332740044, + 0.3938110555196154, + 0.4628208676716171, + 0.7177468630079228, + 1e-06, + 0.03103759015348271, + 0.02828865844409628, + 0.7164333963112022, + 0.19212642197808302, + 0.697210408452911, + 0.7046809935624476, + 0.9225902325860902, + 1.4625751985388367, + 1.0522456240259936, + 0.6818443950002842, + 0.2884065822442686, + 0.5388119913722541, + 0.7247091653772777, + 0.8579675721375568, + 1.1446825413579313, + 1.0496567177779326, + 0.5457951029759756, + 0.2892210779882745, + 0.6832853149096569, + 0.5792173381920694, + 0.7114807110238744, + 1.1989477833750786, + 0.9428868665587309, + 0.5680013204490734, + 0.13264927674671176, + 0.533723969267122, + 0.5334099008762095, + 0.595361087797577, + 0.9635514317546396, + 0.8610451069806081, + 0.608332853444767, + 0.2398860798282785, + 0.4134127040051782, + 0.5671081498003836, + 0.5583210153106908, + 0.8175468104254054, + 0.8285760589919584, + 0.43067805460220965, + 0.2244247227136431, + 0.5358323216821982, + 0.4479674302815383, + 0.5293835095689982, + 0.7832236132310876, + 0.7496773692207523 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 2304, + "num_fit_rows": 2304, + "num_pairwise_rows": 0, + "num_pointwise_rows": 2304, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 8, + "num_calibration_candidates": 2304, + "num_calibration_rows": 288, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5406537007023231, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.0068575279057281335, + "accuracy": 0.5377331420373027, + "confidence": 0.5308756141315746, + "count": 6970, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.04403994227958885, + "accuracy": 0.5857142857142857, + "confidence": 0.6297542279938746, + "count": 420, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.07310076984790292, + "accuracy": 0.6428571428571429, + "confidence": 0.7159579127050458, + "count": 14, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.009091998384416941, + "mean_confidence": 0.5368345782053449, + "n_bins": 10, + "num_pairs": 7404, + "num_rows": 288 + }, + "eval": { + "accuracy": 0.5289459311851448, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.012491333942682648, + "accuracy": 0.520752463421917, + "confidence": 0.5332437973645996, + "count": 3349, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.031039079656006052, + "accuracy": 0.5979020979020979, + "confidence": 0.628941177558104, + "count": 286, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.08836591446494813, + "accuracy": 0.8148148148148148, + "confidence": 0.7264489003498666, + "count": 27, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.01449932655549304, + "mean_confidence": 0.5421422103946226, + "n_bins": 10, + "num_pairs": 3662, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5193384170444694, + "pairwise_causal_calibration_ece": 0.3764812741873265, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14968836800979451, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "seed": "2012211287/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7166993021965027, + "selected_utility_gain_over_base": -0.011547386646270752, + "selector_regret": 0.011076092720031738, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.532810869430912, + "pairwise_causal_calibration_ece": 0.06852515514519775, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.053124156513879084, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "seed": "307068879/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19984664022922516, + "selected_utility_gain_over_base": -0.009305715560913086, + "selector_regret": 0.0029561370611190796, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.515023875986096, + "pairwise_causal_calibration_ece": 0.09211898115676109, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11698321977507692, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "seed": "1073161778/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7180102467536926, + "selected_utility_gain_over_base": -0.007507741451263428, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5408065606528176, + "pairwise_causal_calibration_ece": 0.18366370350996067, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21949047792385512, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "seed": "608016290/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.529986560344696, + "selected_utility_gain_over_base": 0.004626452922821045, + "selector_regret": 0.08247500658035278, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5471109523798312, + "pairwise_causal_calibration_ece": 0.09574619047731159, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07382839572805955, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "seed": "441046872/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.203963503241539, + "selected_utility_gain_over_base": -0.005804136395454407, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.6068801363670007, + "pairwise_causal_calibration_ece": 0.17883414934728506, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2574720584344846, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "seed": "1522272686/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "task_id": "PickCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.46805085413062614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.54236175210767, + "pairwise_causal_calibration_ece": 0.07354190388407468, + "pairwise_causal_calibration_pairs": 7.0, + "predicted_margin": -0.07100608326116234, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5429040222614606, + "pairwise_causal_calibration_ece": 0.2070959777385393, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2893398512416173, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "seed": "1461069713/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5616496449009567, + "pairwise_causal_calibration_ece": 0.11692178367047186, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5432943918031136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.044337254017591476, + "seed": "626486813/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04018332436680794, + "selected_utility_gain_over_base": 0.007489640265703201, + "selector_regret": 0.004153929650783539, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2090834565460682, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5729749711313679, + "pairwise_causal_calibration_ece": 0.11243262732226722, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.5116474609815591, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14525407552719116, + "seed": "2054899126/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8547459244728088, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5369764428991062, + "pairwise_causal_calibration_ece": 0.17730927138660812, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.004905813304151241, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "seed": "672328262/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.045009810477495193, + "selected_utility_gain_over_base": -0.060924429446458817, + "selector_regret": 0.014783523976802826, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5577692830888494, + "pairwise_causal_calibration_ece": 0.1922307169111505, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30034312959483767, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2748139500617981, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.24293343722820282, + "selected_utility_gain_over_base": 0.04594218730926514, + "selector_regret": 0.031880512833595276, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5670465168048365, + "pairwise_causal_calibration_ece": 0.04009634033802069, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11677340499733613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14440619945526123, + "seed": "278008524/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1317293643951416, + "selected_utility_gain_over_base": -0.11828505992889404, + "selector_regret": 0.012676835060119629, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0834110826253891, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5100440403448724, + "pairwise_causal_calibration_ece": 0.13281310251227052, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25691250884199157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "seed": "1845489013/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035594623535871506, + "selected_utility_gain_over_base": -0.0769633837044239, + "selector_regret": 0.007289845496416092, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.03571428571428571, + "pairwise_causal_calibration_confidence": 0.5340533614345875, + "pairwise_causal_calibration_ece": 0.4983390757203018, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13200388445161804, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07115759700536728, + "seed": "626486813/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03407871723175049, + "selected_utility_gain_over_base": -0.01670938730239868, + "selector_regret": 0.03707887977361679, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.006586454808712006, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5549887059926719, + "pairwise_causal_calibration_ece": 0.19501129400732814, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17796449784654716, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.011163711547851562, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.011163711547851562, + "selected_utility_gain_over_base": -1.9888362884521484, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2515503764152527, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5640865527534807, + "pairwise_causal_calibration_ece": 0.11448487581794778, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3088799361849384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "seed": "585904130/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14155812561511993, + "selected_utility_gain_over_base": 0.07052158564329147, + "selector_regret": 0.022925466299057007, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5578282367528864, + "pairwise_causal_calibration_ece": 0.08947433730730506, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4190457481175329, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "seed": "141984615/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01308689545840025, + "selected_utility_gain_over_base": -0.1547458367422223, + "selector_regret": 0.010632594116032124, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5443111304732339, + "pairwise_causal_calibration_ece": 0.11367382662188895, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.20241512494691333, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25842753052711487, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.19032493233680725, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7415724694728851, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5496721466025628, + "pairwise_causal_calibration_ece": 0.15681500374541976, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6023100427879817, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "seed": "636680137/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04850691556930542, + "selected_utility_gain_over_base": -0.07106667757034302, + "selector_regret": 0.05927673727273941, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5348902829449282, + "pairwise_causal_calibration_ece": 0.17939543134078606, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5272354530309642, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.029627859592437744, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.027129650115966797, + "selected_utility_gain_over_base": -0.07471412420272827, + "selector_regret": 0.0024982094764709473, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.16747966408729553, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 1.0, + "pairwise_causal_calibration_confidence": 0.5999128326661309, + "pairwise_causal_calibration_ece": 0.4000871673338691, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.9897121889793562, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5628942850824857, + "pairwise_causal_calibration_ece": 0.061086876087235106, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12575119573418914, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "seed": "2038594191/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08836108446121216, + "selected_utility_gain_over_base": -0.006679512560367584, + "selector_regret": 0.06411068141460419, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5346040083812454, + "pairwise_causal_calibration_ece": 0.17968170590446886, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12103624175712413, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13244229555130005, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09525207430124283, + "selected_utility_gain_over_base": -0.11815530806779861, + "selector_regret": 0.03719022125005722, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09154719114303589, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.4543001715180678, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5257182607095816, + "pairwise_causal_calibration_ece": 0.00999602500470409, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.952906533401014, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "seed": "747260437/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.7486173632887367, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.6011992431167774, + "pairwise_causal_calibration_ece": 0.18451504259750837, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4839704193432687, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21791090071201324, + "selected_utility_gain_over_base": 0.04541558027267456, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5302641729509556, + "pairwise_causal_calibration_ece": 0.18402154133475868, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0849388442130512, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "seed": "1822210471/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035609763115644455, + "selected_utility_gain_over_base": -0.05881569907069206, + "selector_regret": 0.0024167299270629883, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5665724634368305, + "pairwise_causal_calibration_ece": 0.14389694555713678, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3806891099380057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23744209110736847, + "seed": "750448296/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7625579088926315, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5325688201837939, + "pairwise_causal_calibration_ece": 0.038859751244777474, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.7148050601008141, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "seed": "440577711/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5780381556995099, + "pairwise_causal_calibration_ece": 0.19552213561759554, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3335400221980923, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "seed": "1324642411/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5453049027370976, + "pairwise_causal_calibration_ece": 0.13844693675904163, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.2520875088855983, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5103233692062079, + "pairwise_causal_calibration_ece": 0.09681948793664918, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.4662863957877161, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "seed": "1261930456/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44, + "pairwise_causal_calibration_confidence": 0.5600415333760875, + "pairwise_causal_calibration_ece": 0.12004153337608771, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -1.0643933226145403, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5347267964365477, + "pairwise_causal_calibration_ece": 0.2132982250079763, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.46015939778469717, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "seed": "1480856810/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5302836142133847, + "pairwise_causal_calibration_ece": 0.29114495721518674, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028349483019357136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "seed": "2061635888/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07642464339733124, + "selected_utility_gain_over_base": -0.009782284498214722, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5186559158413719, + "pairwise_causal_calibration_ece": 0.33848694130148516, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028816509714223226, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "seed": "655884605/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14302301406860352, + "selected_utility_gain_over_base": -0.0010523796081542969, + "selector_regret": 0.014375895261764526, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5335033957491659, + "pairwise_causal_calibration_ece": 0.03350339574916594, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11300812099822777, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0680118203163147, + "selected_utility_gain_over_base": -0.043087467551231384, + "selector_regret": 0.0060281902551651, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5309964278670222, + "pairwise_causal_calibration_ece": 0.21900357213297783, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3729341213514183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "seed": "1578990937/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21093030273914337, + "selected_utility_gain_over_base": -0.0033356696367263794, + "selector_regret": 0.003326788544654846, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.28, + "pairwise_causal_calibration_confidence": 0.5190763569223411, + "pairwise_causal_calibration_ece": 0.23907635692234108, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.189222442683031, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5206119271327049, + "pairwise_causal_calibration_ece": 0.05081664429586652, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17393811200736525, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "seed": "211245842/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16008998453617096, + "selected_utility_gain_over_base": -0.02421446144580841, + "selector_regret": 0.42213819921016693, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5219633914019057, + "pairwise_causal_calibration_ece": 0.04946518002666567, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22732175807375893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "seed": "1151410346/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22687169909477234, + "selected_utility_gain_over_base": -0.0016018152236938477, + "selector_regret": 0.34162279963493347, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5398428866463935, + "pairwise_causal_calibration_ece": 0.03158568478217802, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21239530032578002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "seed": "1324133122/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05331718921661377, + "selected_utility_gain_over_base": -0.030909866094589233, + "selector_regret": 0.009534716606140137, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5330187864606177, + "pairwise_causal_calibration_ece": 0.17587592931776058, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.01726372954718114, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "seed": "701667486/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6136738657951355, + "selected_utility_gain_over_base": -0.1776002049446106, + "selector_regret": 0.17424744367599487, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5329987081920804, + "pairwise_causal_calibration_ece": 0.14014156533493755, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03891331669299969, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.15669304132461548, + "selected_utility_gain_over_base": -1.8433069586753845, + "selector_regret": 1.8433069586753845, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5428168971264415, + "pairwise_causal_calibration_ece": 0.007102611412155846, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06187932823477338, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "seed": "1999022574/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5931925177574158, + "selected_utility_gain_over_base": -0.23477637767791748, + "selector_regret": 0.0038532018661499023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48148148148148145, + "pairwise_causal_calibration_confidence": 0.5195673663835161, + "pairwise_causal_calibration_ece": 0.03808588490203463, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.03517960107744428, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.719451367855072, + "selected_utility_gain_over_base": -0.008795320987701416, + "selector_regret": 1.280548632144928, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5310502088948758, + "pairwise_causal_calibration_ece": 0.11180693396226715, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0009841947897497316, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "seed": "307068879/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20222051441669464, + "selected_utility_gain_over_base": -0.0069318413734436035, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5164447494303727, + "pairwise_causal_calibration_ece": 0.019269536283913036, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.024067035012088067, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "seed": "1073161778/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7112008929252625, + "selected_utility_gain_over_base": -0.014317095279693604, + "selector_regret": 0.0022280216217041016, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5321689975615193, + "pairwise_causal_calibration_ece": 0.3893118547043764, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08822119814399709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "seed": "608016290/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5021255612373352, + "selected_utility_gain_over_base": -0.023234546184539795, + "selector_regret": 0.10797518491744995, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5207271401629574, + "pairwise_causal_calibration_ece": 0.08641571697989969, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1291832189884023, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "seed": "441046872/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5357279663023194, + "pairwise_causal_calibration_ece": 1.368058803374872e-05, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.8824130953826943, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "task_id": "PickCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.050885567292246775, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5548430273365502, + "pairwise_causal_calibration_ece": 0.13112809452329233, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.7796836425861826, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5353090444202274, + "pairwise_causal_calibration_ece": 0.14326238415120118, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.26002782677736214, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "seed": "1461069713/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05644957348704338, + "selected_utility_gain_over_base": -0.17207801714539528, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5700908567328946, + "pairwise_causal_calibration_ece": 0.14419485755281952, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5780617764704534, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06662949174642563, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05662526562809944, + "selected_utility_gain_over_base": 0.023931581526994705, + "selector_regret": 0.010004226118326187, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18679121881723404, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5546822433674421, + "pairwise_causal_calibration_ece": 0.197539386224585, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0738522840163181, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1511060744524002, + "seed": "2054899126/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12261688709259033, + "selected_utility_gain_over_base": -1.8773831129074097, + "selector_regret": 0.028489187359809875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8488939255475998, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5336834639813024, + "pairwise_causal_calibration_ece": 0.2479691782670167, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15584019094451163, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "seed": "672328262/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5521629142638284, + "pairwise_causal_calibration_ece": 0.05497994287902888, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.10353594341468575, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2738197147846222, + "seed": "767747424/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07682846486568451, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5654063126919278, + "pairwise_causal_calibration_ece": 0.11316511587950095, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14769465328208542, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24920666217803955, + "seed": "278008524/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.24920666217803955, + "selected_utility_gain_over_base": -0.0008077621459960938, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5186589916137023, + "pairwise_causal_calibration_ece": 0.23294470589941663, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10389427792008576, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "seed": "1845489013/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06991276890039444, + "selected_utility_gain_over_base": -0.04264523833990097, + "selector_regret": 0.025380156934261322, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5502200298303185, + "pairwise_causal_calibration_ece": 0.12835139874111012, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4762051028659301, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05724402144551277, + "seed": "626486813/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05123601481318474, + "selected_utility_gain_over_base": 0.00044791027903556824, + "selector_regret": 0.0060080066323280334, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.020500030368566513, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5358042203490166, + "pairwise_causal_calibration_ece": 0.07151850606330235, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.18473664111184027, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.013422667980194092, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24929141998291016, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.541221456647389, + "pairwise_causal_calibration_ece": 0.3269357423616748, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14014647023712193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "seed": "585904130/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08595272153615952, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5379283824128914, + "pairwise_causal_calibration_ece": 0.18078552527003428, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15347700374980155, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "seed": "141984615/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03492546081542969, + "selected_utility_gain_over_base": -0.13290727138519287, + "selector_regret": 0.009052593261003494, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5717256834204711, + "pairwise_causal_calibration_ece": 0.32113145943667165, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.44996305136893266, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2347063571214676, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.16660375893115997, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7652936428785324, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5456989902942814, + "pairwise_causal_calibration_ece": 0.11712756172285277, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6318863252856292, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "seed": "636680137/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07379839569330215, + "selected_utility_gain_over_base": -0.04577519744634628, + "selector_regret": 0.038734301924705505, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5509792993492687, + "pairwise_causal_calibration_ece": 0.265265013634983, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3863417532806279, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04315249249339104, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030954698100686073, + "selected_utility_gain_over_base": -0.070889076218009, + "selector_regret": 0.012197794392704964, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15395503118634224, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5513845519551138, + "pairwise_causal_calibration_ece": 0.2343297337591719, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.5584765371065843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "seed": "1375492786/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5410364066615194, + "pairwise_causal_calibration_ece": 0.21960783523294797, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06608733752201157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15308313071727753, + "selected_utility_gain_over_base": 0.058042533695697784, + "selector_regret": 0.03719758987426758, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5502873058130131, + "pairwise_causal_calibration_ece": 0.12171587724158459, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5180570228476492, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23850953578948975, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1983291506767273, + "selected_utility_gain_over_base": -0.015078231692314148, + "selector_regret": 0.04018038511276245, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.3856860731077444, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5242087065846202, + "pairwise_causal_calibration_ece": 0.15436272198680845, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.8918433784077185, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.5698832828302552, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.6130514300483924, + "pairwise_causal_calibration_ece": 0.31551999852303597, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6249019311477833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.3088321089744568, + "selected_utility_gain_over_base": 0.1363367885351181, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5270692288887334, + "pairwise_causal_calibration_ece": 0.09849780031730487, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.12085404649382042, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5571063160293517, + "pairwise_causal_calibration_ece": 0.271392030315066, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0665134553733544, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2027529925107956, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.18111401796340942, + "selected_utility_gain_over_base": -1.8188859820365906, + "selector_regret": 0.02163897454738617, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7972470074892044, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07692307692307693, + "pairwise_causal_calibration_confidence": 0.5300053688111541, + "pairwise_causal_calibration_ece": 0.4530822918880772, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.9185643919214089, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25925925925925924, + "pairwise_causal_calibration_confidence": 0.5689072335846573, + "pairwise_causal_calibration_ece": 0.3096479743253979, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": -0.28540676920232155, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5203995406793089, + "pairwise_causal_calibration_ece": 0.15817188789211967, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.1676288743355996, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "seed": "1790188787/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5262706089236061, + "pairwise_causal_calibration_ece": 0.06198489463789181, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.4420682590576823, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "seed": "1261930456/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5391718475176117, + "pairwise_causal_calibration_ece": 0.07481690519404037, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.042695474312121, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5393866728393286, + "pairwise_causal_calibration_ece": 0.07510095855361425, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.775305326035764, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5408950551165522, + "pairwise_causal_calibration_ece": 0.17339065916916208, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21267963182785404, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "seed": "2061635888/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08908586204051971, + "selected_utility_gain_over_base": 0.002878934144973755, + "selector_regret": 0.0013788342475891113, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5209751256013208, + "pairwise_causal_calibration_ece": 0.1995465541727493, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03719097122411977, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "seed": "655884605/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12636350095272064, + "selected_utility_gain_over_base": -0.01771189272403717, + "selector_regret": 0.024023786187171936, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5290117220540342, + "pairwise_causal_calibration_ece": 0.10044029348260569, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12608184102453235, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "seed": "1113371261/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09622243046760559, + "selected_utility_gain_over_base": -0.01487685739994049, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.5395331577195327, + "pairwise_causal_calibration_ece": 0.3890382708518959, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30002172815923867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "seed": "1578990937/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21256697177886963, + "selected_utility_gain_over_base": -0.001699000597000122, + "selector_regret": 0.00506339967250824, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.72, + "pairwise_causal_calibration_confidence": 0.5170671799261125, + "pairwise_causal_calibration_ece": 0.20293282007388747, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.2764527397331097, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5442178751645994, + "pairwise_causal_calibration_ece": 0.17006783912111498, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16983155866909005, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "seed": "211245842/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17465433478355408, + "selected_utility_gain_over_base": -0.009650111198425293, + "selector_regret": 0.027022019028663635, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5191499717816545, + "pairwise_causal_calibration_ece": 0.1620071146387973, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10383730061951357, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "seed": "1151410346/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2249394953250885, + "selected_utility_gain_over_base": -0.0035340189933776855, + "selector_regret": 0.01898777484893799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5205020188672198, + "pairwise_causal_calibration_ece": 0.015212266847065892, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21539940459619855, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "seed": "1324133122/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0843733698129654, + "selected_utility_gain_over_base": 0.00014631450176239014, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5271169408929481, + "pairwise_causal_calibration_ece": 0.027116940892948138, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14502256377520684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "seed": "701667486/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.518827017198031, + "pairwise_causal_calibration_ece": 0.054541302912316736, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.21560261538347625, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5236071274921709, + "pairwise_causal_calibration_ece": 0.23789284177788517, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2099363129546052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5207899434564643, + "pairwise_causal_calibration_ece": 0.34221851488503574, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0952851317844515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "seed": "2012211287/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.721496045589447, + "selected_utility_gain_over_base": -0.006750643253326416, + "selector_regret": 0.00554656982421875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5426502518804291, + "pairwise_causal_calibration_ece": 0.042650251880429216, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0658010800556802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "seed": "307068879/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20032821595668793, + "selected_utility_gain_over_base": -0.008824139833450317, + "selector_regret": 0.003201991319656372, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5168209602215624, + "pairwise_causal_calibration_ece": 0.19539238879299098, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.005081361506452997, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7205738425254822, + "selected_utility_gain_over_base": -0.004944145679473877, + "selector_regret": 1.2794261574745178, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5421874092011935, + "pairwise_causal_calibration_ece": 0.32790169491547927, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.23922942899090938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "seed": "608016290/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.38905441761016846, + "selected_utility_gain_over_base": -0.13630568981170654, + "selector_regret": 0.18145644664764404, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5494241983519527, + "pairwise_causal_calibration_ece": 0.2720043730766188, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06231616583540775, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "seed": "441046872/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2042645812034607, + "selected_utility_gain_over_base": -0.005503058433532715, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "task_id": "LiftPegUpright-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5218897792626456, + "pairwise_causal_calibration_ece": 0.22811022073735443, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.5831727753564718, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1522272686/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PickCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.21223227940792033, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5380446434799925, + "pairwise_causal_calibration_ece": 0.06625899934466037, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.6845771853525582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5652627006295361, + "pairwise_causal_calibration_ece": 0.17299321545306823, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.866998389422337, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "seed": "1461069713/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11542954295873642, + "selected_utility_gain_over_base": -0.11309804767370224, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.6208713630599373, + "pairwise_causal_calibration_ece": 0.2661418915792151, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.8909771037624048, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10755517333745956, + "seed": "626486813/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10199248790740967, + "selected_utility_gain_over_base": 0.06929880380630493, + "selector_regret": 0.005562685430049896, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1458655372262001, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5766922131739299, + "pairwise_causal_calibration_ece": 0.13759350111178412, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.39247181169508555, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13674892485141754, + "seed": "2054899126/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8632510751485825, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5319707668349153, + "pairwise_causal_calibration_ece": 0.17482790969205814, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11650077283620158, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "seed": "672328262/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0718381404876709, + "selected_utility_gain_over_base": -0.03409609943628311, + "selector_regret": 0.00032808631658554077, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5354669595034361, + "pairwise_causal_calibration_ece": 0.03596161192513539, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.020965576435658995, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25252875685691833, + "seed": "767747424/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25252875685691833, + "selected_utility_gain_over_base": 0.05553750693798065, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5383303226297562, + "pairwise_causal_calibration_ece": 0.06881253451310088, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.19868620977504126, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.31406447291374207, + "seed": "278008524/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06405004858970642, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5135058626142434, + "pairwise_causal_calibration_ece": 0.049220148328529056, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.058128320354532526, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "seed": "1845489013/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07725050300359726, + "selected_utility_gain_over_base": -0.03530750423669815, + "selector_regret": 0.008944690227508545, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5547158052914759, + "pairwise_causal_calibration_ece": 0.304715805291476, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4392962238663013, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05257529020309448, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.037068385630846024, + "selected_utility_gain_over_base": -0.013719718903303146, + "selector_regret": 0.015506904572248459, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.025168761610984802, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5438217798543279, + "pairwise_causal_calibration_ece": 0.035885276548710696, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13791753681028526, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.01747530698776245, + "seed": "767747424/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.015278160572052002, + "selected_utility_gain_over_base": -1.984721839427948, + "selector_regret": 0.0021971464157104492, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2452387809753418, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5622977631095711, + "pairwise_causal_calibration_ece": 0.13372633453814248, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5298559867159688, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13704223930835724, + "selected_utility_gain_over_base": 0.06600569933652878, + "selector_regret": 0.15684272348880768, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5626061019037739, + "pairwise_causal_calibration_ece": 0.4197489590466311, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.41265012485850616, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "seed": "141984615/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03257067874073982, + "selected_utility_gain_over_base": -0.13526205345988274, + "selector_regret": 0.011906486004590988, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.6161166116060023, + "pairwise_causal_calibration_ece": 0.1472475651230632, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6436743989479843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.4033660888671875, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.4033660888671875, + "selected_utility_gain_over_base": 0.3352634906768799, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.5966339111328125, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5764597095564609, + "pairwise_causal_calibration_ece": 0.1499974907415182, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7477968236574296, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "seed": "636680137/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08920355886220932, + "selected_utility_gain_over_base": -0.030370034277439117, + "selector_regret": 0.03550797700881958, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5418765757613458, + "pairwise_causal_calibration_ece": 0.22044800433277448, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30713097314888393, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053976815193891525, + "seed": "767747424/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032663602381944656, + "selected_utility_gain_over_base": -0.06918017193675041, + "selector_regret": 0.02131321281194687, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14313070848584175, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5465035011084322, + "pairwise_causal_calibration_ece": 0.23921078460585363, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.9167380947098389, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "seed": "1375492786/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5590374392784875, + "pairwise_causal_calibration_ece": 0.05880559485727592, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.26386124774663755, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "seed": "2038594191/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10970946401357651, + "selected_utility_gain_over_base": 0.014668866991996765, + "selector_regret": 0.1390988454222679, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.55655908361266, + "pairwise_causal_calibration_ece": 0.02465314221313835, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.24785714709556195, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17124910652637482, + "seed": "750448296/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05274038016796112, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.11128570670692614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5102074375738617, + "pairwise_causal_calibration_ece": 0.11735029471671882, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.738412396559485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "seed": "747260437/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.5141108613076574, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5595699588623118, + "pairwise_causal_calibration_ece": 0.05956995886231174, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1195884497983013, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "seed": "1549324805/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04154062271118164, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5271103602141303, + "pairwise_causal_calibration_ece": 0.4199675030712731, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.015212697452195612, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "seed": "1822210471/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04166736081242561, + "selected_utility_gain_over_base": -0.052758101373910904, + "selector_regret": 0.017152011394500732, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.55961226629214, + "pairwise_causal_calibration_ece": 0.2738979805778543, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.33354962605897076, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1698276251554489, + "seed": "750448296/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.830172374844551, + "task_id": "PullCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.5741772066455673, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5520567764502703, + "pairwise_causal_calibration_ece": 0.08422908277413624, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": -0.5068909378194302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.5253056653691316, + "pairwise_causal_calibration_ece": 0.06560342553995935, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.1440144908506833, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5549163271210246, + "pairwise_causal_calibration_ece": 0.16205918426388172, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.2751386776848153, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "seed": "1261930456/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3181818181818182, + "pairwise_causal_calibration_confidence": 0.5207948539979985, + "pairwise_causal_calibration_ece": 0.20261303581618034, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.0166419733394694, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5353643069980405, + "pairwise_causal_calibration_ece": 0.0717785501448166, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.7567849205182233, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "seed": "1480856810/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "task_id": "PushCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5213600808814732, + "pairwise_causal_calibration_ece": 0.08578277626138386, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10091080743716707, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "seed": "2061635888/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07194386422634125, + "selected_utility_gain_over_base": -0.014263063669204712, + "selector_regret": 0.00708027184009552, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5269018603789154, + "pairwise_causal_calibration_ece": 0.3126161460932011, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09697672498833285, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "seed": "655884605/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1574591100215912, + "selected_utility_gain_over_base": 0.013383716344833374, + "selector_regret": 0.014966011047363281, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5303568147094659, + "pairwise_causal_calibration_ece": 0.14821461386196266, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16044763104706833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "seed": "1113371261/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09743154048919678, + "selected_utility_gain_over_base": -0.013667747378349304, + "selector_regret": 0.0029332488775253296, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5467275958451122, + "pairwise_causal_calibration_ece": 0.07656675457436006, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30083926520305015, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "seed": "1578990937/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20704451203346252, + "selected_utility_gain_over_base": -0.0072214603424072266, + "selector_regret": 0.025547489523887634, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5137167555753096, + "pairwise_causal_calibration_ece": 0.021997530138976096, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.36227529869707314, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "seed": "1509980664/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5332794039871745, + "pairwise_causal_calibration_ece": 0.18100631029853975, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09204961172655701, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "seed": "211245842/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18301993608474731, + "selected_utility_gain_over_base": -0.0012845098972320557, + "selector_regret": 0.031265079975128174, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5294358690784804, + "pairwise_causal_calibration_ece": 0.3634212737786625, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.26548608597987267, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "seed": "1151410346/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.24776706099510193, + "selected_utility_gain_over_base": 0.019293546676635742, + "selector_regret": 0.31524333357810974, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5163907247856556, + "pairwise_causal_calibration_ece": 0.05503784664291578, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06331113876354011, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "seed": "1324133122/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0906635969877243, + "selected_utility_gain_over_base": 0.006436541676521301, + "selector_regret": 0.007487863302230835, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5417148439395401, + "pairwise_causal_calibration_ece": 0.0699820652511001, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.17712518900425797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5238928346234378, + "pairwise_causal_calibration_ece": 0.011821451090847868, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.22361307170053446, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "seed": "412102563/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5379278533729487, + "pairwise_causal_calibration_ece": 0.2835007180556227, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1638733757665311, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "seed": "1999022574/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "task_id": "StackCube-v1", + "tau": -0.09621934768667256, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 10.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3958333333333333, + "low": 0.2277777777777778, + "mean": 0.3083333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3083333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8053333333333332, + "low": 0.10836363636363637, + "mean": 0.45684848484848484, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.45684848484848484, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.4097222222222222, + "low": 0.2569444444444444, + "mean": 0.3333333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9566364926286041, + "low": 0.6609309806488455, + "mean": 0.8028034761858482, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.8028034761858482, + "per_seed": { + "1073161778/train0": 0.725517988204956, + "1073161778/train1": 0.725517988204956, + "1073161778/train2": 0.725517988204956, + "1113371261/train0": 0.11109928786754608, + "1113371261/train1": 0.11109928786754608, + "1113371261/train2": 0.11109928786754608, + "1151410346/train0": 0.2284735143184662, + "1151410346/train1": 0.2284735143184662, + "1151410346/train2": 0.2284735143184662, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.084227055311203, + "1324133122/train2": 0.084227055311203, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.16783273220062256, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.22852759063243866, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.17249532043933868, + "1549324805/train1": 0.17249532043933868, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21426597237586975, + "1578990937/train1": 0.21426597237586975, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.09442546218633652, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.09442546218633652, + "1845489013/train0": 0.11255800724029541, + "1845489013/train1": 0.11255800724029541, + "1845489013/train2": 0.11255800724029541, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.09504059702157974, + "2038594191/train1": 0.09504059702157974, + "2038594191/train2": 0.09504059702157974, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.08620692789554596, + "2061635888/train1": 0.08620692789554596, + "2061635888/train2": 0.08620692789554596, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.1250072121620178, + "278008524/train1": 1.1250072121620178, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.20915235579013824, + "307068879/train1": 0.20915235579013824, + "307068879/train2": 0.20915235579013824, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.2097676396369934, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2097676396369934, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.07103653997182846, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.07103653997182846, + "608016290/train0": 0.525360107421875, + "608016290/train1": 0.525360107421875, + "608016290/train2": 0.525360107421875, + "626486813/train0": 0.04174089431762695, + "626486813/train1": 0.04174089431762695, + "626486813/train2": 0.04174089431762695, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.1440753936767578, + "655884605/train2": 0.1440753936767578, + "672328262/train0": 0.10593423992395401, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.10593423992395401, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7605033268531164, + "750448296/train1": 0.7605033268531164, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0747087560594082, + "767747424/train1": 1.0747087560594082, + "767747424/train2": 1.0747087560594082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6347634333101186, + "low": 0.5526382644826715, + "mean": 1.0913353469100866, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.0913353469100866, + "per_task": { + "LiftPegUpright-v1": 0.4796089559793472, + "PickCube-v1": 2.0, + "PullCube-v1": 0.8032506120204925, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.5003613992170854 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9699174793851044, + "low": 0.6952519603073597, + "mean": 0.8338337515791258, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6833333333333333, + "low": 0.5118055555555556, + "mean": 0.5986111111111111, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5986111111111111, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 1.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 1.0, + "767747424/train1": 0.25, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7975757575757576, + "low": 0.13333333333333333, + "mean": 0.46545454545454545, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.46545454545454545, + "per_task": { + "LiftPegUpright-v1": 0.9333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.6666666666666666, + "PushCube-v1": 0.0, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.6944444444444444, + "low": 0.5347222222222222, + "mean": 0.6111111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6833333333333333, + "low": 0.5118055555555556, + "mean": 0.5986111111111111, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5986111111111111, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 1.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 1.0, + "767747424/train1": 0.25, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7975757575757576, + "low": 0.13333333333333333, + "mean": 0.46545454545454545, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.46545454545454545, + "per_task": { + "LiftPegUpright-v1": 0.9333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.6666666666666666, + "PushCube-v1": 0.0, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.6944444444444444, + "low": 0.5347222222222222, + "mean": 0.6111111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.48819444444444443, + "low": 0.31666666666666665, + "mean": 0.40138888888888885, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.40138888888888885, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 1.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 1.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.0, + "767747424/train1": 0.75, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8666666666666666, + "low": 0.2024242424242424, + "mean": 0.5345454545454545, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5345454545454545, + "per_task": { + "LiftPegUpright-v1": 0.06666666666666667, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 1.0, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.4652777777777778, + "low": 0.3055555555555556, + "mean": 0.3888888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.45138888888888895, + "low": 0.2833333333333333, + "mean": 0.3631944444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3631944444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 1.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4609696969696969, + "low": 0.1, + "mean": 0.2852121212121212, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2852121212121212, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.6060606060606061 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.4444444444444444, + "low": 0.2847222222222222, + "mean": 0.3611111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5635216491682006, + "low": 0.49323070846777733, + "mean": 0.5272910574419193, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.5272910574419193, + "per_seed": { + "1073161778/train0": 0.6071428571428571, + "1073161778/train1": 0.5357142857142857, + "1073161778/train2": 0.32142857142857145, + "1113371261/train0": 0.5, + "1113371261/train1": 0.42857142857142855, + "1113371261/train2": 0.6785714285714286, + "1151410346/train0": 0.5714285714285714, + "1151410346/train1": 0.35714285714285715, + "1151410346/train2": 0.8928571428571429, + "1261930456/train0": 0.6071428571428571, + "1261930456/train1": 0.4642857142857143, + "1261930456/train2": 0.39285714285714285, + "1324133122/train0": 0.5714285714285714, + "1324133122/train1": 0.5357142857142857, + "1324133122/train2": 0.5714285714285714, + "1324642411/train0": 0.4642857142857143, + "1324642411/train1": 0.25925925925925924, + "1324642411/train2": 0.5555555555555556, + "1375492786/train0": 1.0, + "1375492786/train1": 0.7857142857142857, + "1375492786/train2": 0.7857142857142857, + "141984615/train0": 0.5714285714285714, + "141984615/train1": 0.35714285714285715, + "141984615/train2": 0.14285714285714285, + "1461069713/train0": 0.75, + "1461069713/train1": 0.6785714285714286, + "1461069713/train2": 0.5714285714285714, + "1480856810/train0": 0.32142857142857145, + "1480856810/train1": 0.4642857142857143, + "1480856810/train2": 0.6071428571428571, + "1509980664/train0": 0.28, + "1509980664/train1": 0.72, + "1509980664/train2": 0.5357142857142857, + "1522272686/train0": 0.7857142857142857, + "1522272686/train1": 0.5357142857142857, + "1522272686/train2": 0.75, + "1549324805/train0": 0.7857142857142857, + "1549324805/train1": 0.9285714285714286, + "1549324805/train2": 0.5, + "1578990937/train0": 0.75, + "1578990937/train1": 0.9285714285714286, + "1578990937/train2": 0.5357142857142857, + "1790188787/train0": 0.6071428571428571, + "1790188787/train1": 0.6785714285714286, + "1790188787/train2": 0.5909090909090909, + "1822210471/train0": 0.7142857142857143, + "1822210471/train1": 0.42857142857142855, + "1822210471/train2": 0.10714285714285714, + "1845489013/train0": 0.6428571428571429, + "1845489013/train1": 0.2857142857142857, + "1845489013/train2": 0.4642857142857143, + "1999022574/train0": 0.5357142857142857, + "1999022574/train1": 0.2857142857142857, + "1999022574/train2": 0.8214285714285714, + "2012211287/train0": 0.14285714285714285, + "2012211287/train1": 0.48148148148148145, + "2012211287/train2": 0.17857142857142858, + "2038594191/train0": 0.5714285714285714, + "2038594191/train1": 0.32142857142857145, + "2038594191/train2": 0.5357142857142857, + "2054899126/train0": 0.5714285714285714, + "2054899126/train1": 0.35714285714285715, + "2054899126/train2": 0.7142857142857143, + "2061635888/train0": 0.8214285714285714, + "2061635888/train1": 0.7142857142857143, + "2061635888/train2": 0.6071428571428571, + "211245842/train0": 0.5714285714285714, + "211245842/train1": 0.7142857142857143, + "211245842/train2": 0.7142857142857143, + "278008524/train0": 0.6071428571428571, + "278008524/train1": 0.6785714285714286, + "278008524/train2": 0.6071428571428571, + "307068879/train0": 0.4642857142857143, + "307068879/train1": 0.6428571428571429, + "307068879/train2": 0.5, + "412102563/train0": 0.39285714285714285, + "412102563/train1": 0.4642857142857143, + "412102563/train2": 0.5357142857142857, + "440577711/train0": 0.5714285714285714, + "440577711/train1": 0.07692307692307693, + "441046872/train0": 0.6428571428571429, + "441046872/train1": 0.6071428571428571, + "441046872/train2": 0.8214285714285714, + "509430392/train0": 0.44, + "509430392/train1": 0.5, + "509430392/train2": 0.3181818181818182, + "585904130/train0": 0.6785714285714286, + "585904130/train1": 0.21428571428571427, + "585904130/train2": 0.42857142857142855, + "608016290/train0": 0.35714285714285715, + "608016290/train1": 0.14285714285714285, + "608016290/train2": 0.21428571428571427, + "626486813/train0": 0.35714285714285715, + "626486813/train1": 0.6964285714285714, + "626486813/train2": 0.3035714285714286, + "636680137/train0": 0.39285714285714285, + "636680137/train1": 0.42857142857142855, + "636680137/train2": 0.4642857142857143, + "655884605/train0": 0.8571428571428571, + "655884605/train1": 0.32142857142857145, + "655884605/train2": 0.21428571428571427, + "672328262/train0": 0.7142857142857143, + "672328262/train1": 0.2857142857142857, + "672328262/train2": 0.35714285714285715, + "701667486/train0": 0.35714285714285715, + "701667486/train1": 0.5, + "701667486/train2": 0.5, + "747260437/train0": 0.5357142857142857, + "747260437/train1": 0.6785714285714286, + "747260437/train2": 0.39285714285714285, + "750448296/train0": 0.6071428571428572, + "750448296/train1": 0.5357142857142857, + "750448296/train2": 0.5238095238095237, + "767747424/train0": 0.6964285714285714, + "767747424/train1": 0.4910714285714285, + "767747424/train2": 0.48214285714285715 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6213079426020601, + "low": 0.4734243447184624, + "mean": 0.5412774792382635, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5412774792382635, + "per_task": { + "LiftPegUpright-v1": 0.44400352733686066, + "PickCube-v1": 0.6904761904761904, + "PullCube-v1": 0.5367965367965366, + "PushCube-v1": 0.4658470723176606, + "StackCube-v1": 0.5692640692640694 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5603762931374873, + "low": 0.49524656658984995, + "mean": 0.5288445330236378, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5444476866516909, + "low": 0.5366940555039924, + "mean": 0.5404222265968583, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.5404222265968583, + "per_seed": { + "1073161778/train0": 0.515023875986096, + "1073161778/train1": 0.5164447494303727, + "1073161778/train2": 0.5168209602215624, + "1113371261/train0": 0.5335033957491659, + "1113371261/train1": 0.5290117220540342, + "1113371261/train2": 0.5303568147094659, + "1151410346/train0": 0.5219633914019057, + "1151410346/train1": 0.5191499717816545, + "1151410346/train2": 0.5294358690784804, + "1261930456/train0": 0.5103233692062079, + "1261930456/train1": 0.5262706089236061, + "1261930456/train2": 0.5549163271210246, + "1324133122/train0": 0.5398428866463935, + "1324133122/train1": 0.5205020188672198, + "1324133122/train2": 0.5163907247856556, + "1324642411/train0": 0.5780381556995099, + "1324642411/train1": 0.5689072335846573, + "1324642411/train2": 0.5520567764502703, + "1375492786/train0": 0.5999128326661309, + "1375492786/train1": 0.5513845519551138, + "1375492786/train2": 0.5465035011084322, + "141984615/train0": 0.5578282367528864, + "141984615/train1": 0.5379283824128914, + "141984615/train2": 0.5626061019037739, + "1461069713/train0": 0.5429040222614606, + "1461069713/train1": 0.5353090444202274, + "1461069713/train2": 0.5652627006295361, + "1480856810/train0": 0.5347267964365477, + "1480856810/train1": 0.5393866728393286, + "1480856810/train2": 0.5353643069980405, + "1509980664/train0": 0.5190763569223411, + "1509980664/train1": 0.5170671799261125, + "1509980664/train2": 0.5137167555753096, + "1522272686/train0": 0.6068801363670007, + "1522272686/train1": 0.5357279663023194, + "1522272686/train2": 0.5218897792626456, + "1549324805/train0": 0.6011992431167774, + "1549324805/train1": 0.6130514300483924, + "1549324805/train2": 0.5595699588623118, + "1578990937/train0": 0.5309964278670222, + "1578990937/train1": 0.5395331577195327, + "1578990937/train2": 0.5467275958451122, + "1790188787/train0": 0.5453049027370976, + "1790188787/train1": 0.5203995406793089, + "1790188787/train2": 0.5253056653691316, + "1822210471/train0": 0.5302641729509556, + "1822210471/train1": 0.5270692288887334, + "1822210471/train2": 0.5271103602141303, + "1845489013/train0": 0.5100440403448724, + "1845489013/train1": 0.5186589916137023, + "1845489013/train2": 0.5135058626142434, + "1999022574/train0": 0.5428168971264415, + "1999022574/train1": 0.5236071274921709, + "1999022574/train2": 0.5379278533729487, + "2012211287/train0": 0.5193384170444694, + "2012211287/train1": 0.5195673663835161, + "2012211287/train2": 0.5207899434564643, + "2038594191/train0": 0.5628942850824857, + "2038594191/train1": 0.5410364066615194, + "2038594191/train2": 0.5590374392784875, + "2054899126/train0": 0.5729749711313679, + "2054899126/train1": 0.5546822433674421, + "2054899126/train2": 0.5766922131739299, + "2061635888/train0": 0.5302836142133847, + "2061635888/train1": 0.5408950551165522, + "2061635888/train2": 0.5213600808814732, + "211245842/train0": 0.5206119271327049, + "211245842/train1": 0.5442178751645994, + "211245842/train2": 0.5332794039871745, + "278008524/train0": 0.5670465168048365, + "278008524/train1": 0.5654063126919278, + "278008524/train2": 0.5383303226297562, + "307068879/train0": 0.532810869430912, + "307068879/train1": 0.5310502088948758, + "307068879/train2": 0.5426502518804291, + "412102563/train0": 0.5329987081920804, + "412102563/train1": 0.518827017198031, + "412102563/train2": 0.5238928346234378, + "440577711/train0": 0.5325688201837939, + "440577711/train1": 0.5300053688111541, + "441046872/train0": 0.5471109523798312, + "441046872/train1": 0.5207271401629574, + "441046872/train2": 0.5494241983519527, + "509430392/train0": 0.5600415333760875, + "509430392/train1": 0.5391718475176117, + "509430392/train2": 0.5207948539979985, + "585904130/train0": 0.5640865527534807, + "585904130/train1": 0.541221456647389, + "585904130/train2": 0.5622977631095711, + "608016290/train0": 0.5408065606528176, + "608016290/train1": 0.5321689975615193, + "608016290/train2": 0.5421874092011935, + "626486813/train0": 0.5478515031677721, + "626486813/train1": 0.5601554432816065, + "626486813/train2": 0.5877935841757066, + "636680137/train0": 0.5496721466025628, + "636680137/train1": 0.5456989902942814, + "636680137/train2": 0.5764597095564609, + "655884605/train0": 0.5186559158413719, + "655884605/train1": 0.5209751256013208, + "655884605/train2": 0.5269018603789154, + "672328262/train0": 0.5369764428991062, + "672328262/train1": 0.5336834639813024, + "672328262/train2": 0.5319707668349153, + "701667486/train0": 0.5330187864606177, + "701667486/train1": 0.5271169408929481, + "701667486/train2": 0.5417148439395401, + "747260437/train0": 0.5257182607095816, + "747260437/train1": 0.5242087065846202, + "747260437/train2": 0.5102074375738617, + "750448296/train0": 0.5484958674304367, + "750448296/train1": 0.559706435087612, + "750448296/train2": 0.5774293205036009, + "767747424/train0": 0.5475025060335299, + "767747424/train1": 0.548447365324666, + "767747424/train2": 0.5398024896497756 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5506570720689693, + "low": 0.5315369922836274, + "mean": 0.5409853593104177, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5409853593104177, + "per_task": { + "LiftPegUpright-v1": 0.5297947934025979, + "PickCube-v1": 0.5548326273106553, + "PullCube-v1": 0.5519987946284919, + "PushCube-v1": 0.5396225164665516, + "StackCube-v1": 0.5286780647437916 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5461624978152934, + "low": 0.5385708694329345, + "mean": 0.5422634264546857, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1741830737429508, + "low": 0.13369653419308117, + "mean": 0.15375344277282504, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.15375344277282504, + "per_seed": { + "1073161778/train0": 0.09211898115676109, + "1073161778/train1": 0.019269536283913036, + "1073161778/train2": 0.19539238879299098, + "1113371261/train0": 0.03350339574916594, + "1113371261/train1": 0.10044029348260569, + "1113371261/train2": 0.14821461386196266, + "1151410346/train0": 0.04946518002666567, + "1151410346/train1": 0.1620071146387973, + "1151410346/train2": 0.3634212737786625, + "1261930456/train0": 0.09681948793664918, + "1261930456/train1": 0.06198489463789181, + "1261930456/train2": 0.16205918426388172, + "1324133122/train0": 0.03158568478217802, + "1324133122/train1": 0.015212266847065892, + "1324133122/train2": 0.05503784664291578, + "1324642411/train0": 0.19552213561759554, + "1324642411/train1": 0.3096479743253979, + "1324642411/train2": 0.08422908277413624, + "1375492786/train0": 0.4000871673338691, + "1375492786/train1": 0.2343297337591719, + "1375492786/train2": 0.23921078460585363, + "141984615/train0": 0.08947433730730506, + "141984615/train1": 0.18078552527003428, + "141984615/train2": 0.4197489590466311, + "1461069713/train0": 0.2070959777385393, + "1461069713/train1": 0.14326238415120118, + "1461069713/train2": 0.17299321545306823, + "1480856810/train0": 0.2132982250079763, + "1480856810/train1": 0.07510095855361425, + "1480856810/train2": 0.0717785501448166, + "1509980664/train0": 0.23907635692234108, + "1509980664/train1": 0.20293282007388747, + "1509980664/train2": 0.021997530138976096, + "1522272686/train0": 0.17883414934728506, + "1522272686/train1": 1.368058803374872e-05, + "1522272686/train2": 0.22811022073735443, + "1549324805/train0": 0.18451504259750837, + "1549324805/train1": 0.31551999852303597, + "1549324805/train2": 0.05956995886231174, + "1578990937/train0": 0.21900357213297783, + "1578990937/train1": 0.3890382708518959, + "1578990937/train2": 0.07656675457436006, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.13844693675904163, + "1790188787/train1": 0.15817188789211967, + "1790188787/train2": 0.06560342553995935, + "1822210471/train0": 0.18402154133475868, + "1822210471/train1": 0.09849780031730487, + "1822210471/train2": 0.4199675030712731, + "1845489013/train0": 0.13281310251227052, + "1845489013/train1": 0.23294470589941663, + "1845489013/train2": 0.049220148328529056, + "1999022574/train0": 0.007102611412155846, + "1999022574/train1": 0.23789284177788517, + "1999022574/train2": 0.2835007180556227, + "2012211287/train0": 0.3764812741873265, + "2012211287/train1": 0.03808588490203463, + "2012211287/train2": 0.34221851488503574, + "2038594191/train0": 0.061086876087235106, + "2038594191/train1": 0.21960783523294797, + "2038594191/train2": 0.05880559485727592, + "2054899126/train0": 0.05621631366113361, + "2054899126/train1": 0.0987696931122925, + "2054899126/train2": 0.06879675055589206, + "2061635888/train0": 0.29114495721518674, + "2061635888/train1": 0.17339065916916208, + "2061635888/train2": 0.08578277626138386, + "211245842/train0": 0.05081664429586652, + "211245842/train1": 0.17006783912111498, + "211245842/train2": 0.18100631029853975, + "278008524/train0": 0.020048170169010344, + "278008524/train1": 0.05658255793975048, + "278008524/train2": 0.03440626725655044, + "307068879/train0": 0.06852515514519775, + "307068879/train1": 0.11180693396226715, + "307068879/train2": 0.042650251880429216, + "412102563/train0": 0.14014156533493755, + "412102563/train1": 0.054541302912316736, + "412102563/train2": 0.011821451090847868, + "440577711/train0": 0.038859751244777474, + "440577711/train1": 0.4530822918880772, + "440577711/train2": 0.0, + "441046872/train0": 0.09574619047731159, + "441046872/train1": 0.08641571697989969, + "441046872/train2": 0.2720043730766188, + "509430392/train0": 0.12004153337608771, + "509430392/train1": 0.07481690519404037, + "509430392/train2": 0.20261303581618034, + "585904130/train0": 0.11448487581794778, + "585904130/train1": 0.3269357423616748, + "585904130/train2": 0.13372633453814248, + "608016290/train0": 0.18366370350996067, + "608016290/train1": 0.3893118547043764, + "608016290/train2": 0.32790169491547927, + "626486813/train0": 0.30763042969538684, + "626486813/train1": 0.13627312814696482, + "626486813/train2": 0.28542884843534555, + "636680137/train0": 0.15681500374541976, + "636680137/train1": 0.11712756172285277, + "636680137/train2": 0.1499974907415182, + "655884605/train0": 0.33848694130148516, + "655884605/train1": 0.1995465541727493, + "655884605/train2": 0.3126161460932011, + "672328262/train0": 0.17730927138660812, + "672328262/train1": 0.2479691782670167, + "672328262/train2": 0.17482790969205814, + "701667486/train0": 0.17587592931776058, + "701667486/train1": 0.027116940892948138, + "701667486/train2": 0.0699820652511001, + "747260437/train0": 0.00999602500470409, + "747260437/train1": 0.15436272198680845, + "747260437/train2": 0.11735029471671882, + "750448296/train0": 0.14575082602783154, + "750448296/train1": 0.23807978899777407, + "750448296/train2": 0.14859956263801863, + "767747424/train0": 0.16004483653583484, + "767747424/train1": 0.13072288927515163, + "767747424/train2": 0.08963847303782024 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.16363791509034706, + "low": 0.13936582703049763, + "mean": 0.15013409075395842, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.15013409075395842, + "per_task": { + "LiftPegUpright-v1": 0.17610616365730683, + "PickCube-v1": 0.13565268355755775, + "PullCube-v1": 0.14975573664802355, + "PushCube-v1": 0.14011534783179128, + "StackCube-v1": 0.14904052207511279 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.1697497867995513, + "low": 0.13174029337093607, + "mean": 0.15083780724711812, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 26.716666666666665, + "low": 24.608333333333334, + "mean": 25.747916666666665, + "n": 120, + "num_samples": 1000 + }, + "mean": 25.747916666666665, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 28.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 27.0, + "1324642411/train2": 18.0, + "1375492786/train0": 13.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 28.0, + "1480856810/train1": 28.0, + "1480856810/train2": 28.0, + "1509980664/train0": 25.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 28.0, + "1790188787/train1": 28.0, + "1790188787/train2": 22.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 28.0, + "1999022574/train1": 28.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 27.0, + "2012211287/train2": 28.0, + "2038594191/train0": 28.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 14.0, + "278008524/train1": 14.0, + "278008524/train2": 14.0, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 28.0, + "412102563/train1": 28.0, + "412102563/train2": 28.0, + "440577711/train0": 28.0, + "440577711/train1": 13.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 25.0, + "509430392/train1": 22.0, + "509430392/train2": 22.0, + "585904130/train0": 28.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 28.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 28.0, + "672328262/train1": 28.0, + "672328262/train2": 28.0, + "701667486/train0": 28.0, + "701667486/train1": 28.0, + "701667486/train2": 28.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 28.0, + "750448296/train1": 28.0, + "750448296/train2": 28.0, + "767747424/train0": 22.75, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 27.927272727272726, + "low": 24.69563636363636, + "mean": 26.348969696969696, + "n": 5, + "num_samples": 1000 + }, + "mean": 26.348969696969696, + "per_task": { + "LiftPegUpright-v1": 27.933333333333334, + "PickCube-v1": 28.0, + "PullCube-v1": 24.16, + "PushCube-v1": 23.833333333333332, + "StackCube-v1": 27.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 26.604166666666668, + "low": 24.104166666666668, + "mean": 25.430555555555557, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.02751201982922565, + "low": -0.20502690372613336, + "mean": -0.11758667271105495, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.11758667271105495, + "per_seed": { + "1073161778/train0": 0.11698321977507692, + "1073161778/train1": 0.024067035012088067, + "1073161778/train2": 0.005081361506452997, + "1113371261/train0": 0.11300812099822777, + "1113371261/train1": 0.12608184102453235, + "1113371261/train2": 0.16044763104706833, + "1151410346/train0": 0.22732175807375893, + "1151410346/train1": 0.10383730061951357, + "1151410346/train2": 0.26548608597987267, + "1261930456/train0": -1.4662863957877161, + "1261930456/train1": -1.4420682590576823, + "1261930456/train2": -1.2751386776848153, + "1324133122/train0": 0.21239530032578002, + "1324133122/train1": 0.21539940459619855, + "1324133122/train2": 0.06331113876354011, + "1324642411/train0": -0.3335400221980923, + "1324642411/train1": -0.28540676920232155, + "1324642411/train2": -0.5068909378194302, + "1375492786/train0": -0.9897121889793562, + "1375492786/train1": -1.5584765371065843, + "1375492786/train2": -0.9167380947098389, + "141984615/train0": 0.4190457481175329, + "141984615/train1": 0.15347700374980155, + "141984615/train2": 0.41265012485850616, + "1461069713/train0": -0.2893398512416173, + "1461069713/train1": 0.26002782677736214, + "1461069713/train2": 0.866998389422337, + "1480856810/train0": -0.46015939778469717, + "1480856810/train1": -0.775305326035764, + "1480856810/train2": -0.7567849205182233, + "1509980664/train0": -0.189222442683031, + "1509980664/train1": -0.2764527397331097, + "1509980664/train2": -0.36227529869707314, + "1522272686/train0": -0.2574720584344846, + "1522272686/train1": -0.8824130953826943, + "1522272686/train2": -0.5831727753564718, + "1549324805/train0": 0.4839704193432687, + "1549324805/train1": 0.6249019311477833, + "1549324805/train2": -0.1195884497983013, + "1578990937/train0": 0.3729341213514183, + "1578990937/train1": 0.30002172815923867, + "1578990937/train2": 0.30083926520305015, + "1747178030/train0": -0.46805085413062614, + "1747178030/train1": -0.050885567292246775, + "1747178030/train2": -0.21223227940792033, + "1790188787/train0": -1.2520875088855983, + "1790188787/train1": -1.1676288743355996, + "1790188787/train2": -1.1440144908506833, + "1822210471/train0": 0.0849388442130512, + "1822210471/train1": -0.12085404649382042, + "1822210471/train2": -0.015212697452195612, + "1845489013/train0": 0.25691250884199157, + "1845489013/train1": 0.10389427792008576, + "1845489013/train2": 0.058128320354532526, + "1999022574/train0": 0.06187932823477338, + "1999022574/train1": -0.2099363129546052, + "1999022574/train2": -0.1638733757665311, + "2012211287/train0": 0.14968836800979451, + "2012211287/train1": 0.03517960107744428, + "2012211287/train2": -0.0952851317844515, + "2038594191/train0": 0.12575119573418914, + "2038594191/train1": -0.06608733752201157, + "2038594191/train2": 0.26386124774663755, + "2054899126/train0": -0.028673644731745662, + "2054899126/train1": 0.15591689454571314, + "2054899126/train2": -0.14059305249407972, + "2061635888/train0": 0.028349483019357136, + "2061635888/train1": 0.21267963182785404, + "2061635888/train2": 0.10091080743716707, + "211245842/train0": 0.17393811200736525, + "211245842/train1": 0.16983155866909005, + "211245842/train2": 0.09204961172655701, + "278008524/train0": 0.43269538414303643, + "278008524/train1": 0.35878896805617033, + "278008524/train2": 0.15771232576630806, + "307068879/train0": -0.053124156513879084, + "307068879/train1": -0.0009841947897497316, + "307068879/train2": -0.0658010800556802, + "412102563/train0": 0.03891331669299969, + "412102563/train1": -0.21560261538347625, + "412102563/train2": -0.22361307170053446, + "440577711/train0": -0.7148050601008141, + "440577711/train1": -0.9185643919214089, + "440577711/train2": -0.5741772066455673, + "441046872/train0": 0.07382839572805955, + "441046872/train1": -0.1291832189884023, + "441046872/train2": 0.06231616583540775, + "509430392/train0": -1.0643933226145403, + "509430392/train1": -1.042695474312121, + "509430392/train2": -1.0166419733394694, + "585904130/train0": 0.3088799361849384, + "585904130/train1": -0.14014647023712193, + "585904130/train2": 0.5298559867159688, + "608016290/train0": 0.21949047792385512, + "608016290/train1": 0.08822119814399709, + "608016290/train2": 0.23922942899090938, + "626486813/train0": 0.3376491381273658, + "626486813/train1": 0.5271334396681917, + "626486813/train2": 0.665136663814353, + "636680137/train0": 0.6023100427879817, + "636680137/train1": 0.6318863252856292, + "636680137/train2": 0.7477968236574296, + "655884605/train0": 0.028816509714223226, + "655884605/train1": -0.03719097122411977, + "655884605/train2": 0.09697672498833285, + "672328262/train0": -0.004905813304151241, + "672328262/train1": -0.15584019094451163, + "672328262/train2": 0.11650077283620158, + "701667486/train0": -0.01726372954718114, + "701667486/train1": -0.14502256377520684, + "701667486/train2": -0.17712518900425797, + "747260437/train0": -0.952906533401014, + "747260437/train1": -0.8918433784077185, + "747260437/train2": -0.738412396559485, + "750448296/train0": -0.15402266437593162, + "750448296/train1": 0.0005268387017873799, + "750448296/train2": 0.02075587526448386, + "767747424/train0": 0.23363424930279666, + "767747424/train1": -0.17040361845802016, + "767747424/train2": -0.06512356295726199 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.04736062755820034, + "low": -0.6394442145473573, + "mean": -0.26676488926688824, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.26676488926688824, + "per_task": { + "LiftPegUpright-v1": 0.044647164658061525, + "PickCube-v1": -0.5743526430578836, + "PullCube-v1": 0.051817275904245494, + "PushCube-v1": -0.8998105005052524, + "StackCube-v1": 0.04387425666638764 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": -0.001422099042780716, + "low": -0.16447177451546782, + "mean": -0.08274856462213934, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.29583333333333334, + "low": 0.15416666666666667, + "mean": 0.22291666666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.22291666666666668, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40606060606060607, + "low": 0.15903030303030302, + "mean": 0.2754747474747475, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2754747474747475, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.3333333333333333, + "PullCube-v1": 0.17333333333333334, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.18181818181818182 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.2916666666666667, + "low": 0.1597222222222222, + "mean": 0.2222222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7715499536289523, + "low": 0.5113214392965246, + "mean": 0.6364790620334033, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6364790620334033, + "per_seed": { + "1073161778/train0": 0.7180102467536926, + "1073161778/train1": 0.7134289145469666, + "1073161778/train2": 2.0, + "1113371261/train0": 0.0740400105714798, + "1113371261/train1": 0.09622243046760559, + "1113371261/train2": 0.1003647893667221, + "1151410346/train0": 0.5684944987297058, + "1151410346/train1": 0.2439272701740265, + "1151410346/train2": 0.5630103945732117, + "1261930456/train0": 0.09678153693675995, + "1261930456/train1": 0.09997129440307617, + "1261930456/train2": 0.08545441925525665, + "1324133122/train0": 0.0628519058227539, + "1324133122/train1": 0.0843733698129654, + "1324133122/train2": 0.09815146028995514, + "1324642411/train0": 0.21000581979751587, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.14743654429912567, + "1375492786/train2": 0.19452734291553497, + "141984615/train0": 0.023719489574432373, + "141984615/train1": 0.04397805407643318, + "141984615/train2": 0.04447716474533081, + "1461069713/train0": 0.05616961047053337, + "1461069713/train1": 0.05644957348704338, + "1461069713/train2": 0.11542954295873642, + "1480856810/train0": 0.14178013801574707, + "1480856810/train1": 2.0, + "1480856810/train2": 0.19622710347175598, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.83889240026474, + "1522272686/train0": 0.5233199596405029, + "1522272686/train1": 0.5422300696372986, + "1522272686/train2": 2.0, + "1549324805/train0": 0.21791090071201324, + "1549324805/train1": 0.3088321089744568, + "1549324805/train2": 0.21403594315052032, + "1578990937/train0": 0.21425709128379822, + "1578990937/train1": 0.21763037145137787, + "1578990937/train2": 0.23259200155735016, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.1862330436706543, + "1790188787/train2": 2.0, + "1822210471/train0": 0.03802649304270744, + "1822210471/train1": 0.07539409399032593, + "1822210471/train2": 0.058819372206926346, + "1845489013/train0": 0.0428844690322876, + "1845489013/train1": 0.09529292583465576, + "1845489013/train2": 0.0861951932311058, + "1999022574/train0": 0.5970457196235657, + "1999022574/train1": 2.0, + "1999022574/train2": 0.7970014810562134, + "2012211287/train0": 0.7277753949165344, + "2012211287/train1": 2.0, + "2012211287/train2": 0.7270426154136658, + "2038594191/train0": 0.15247176587581635, + "2038594191/train1": 0.1902807205915451, + "2038594191/train2": 0.24880830943584442, + "2054899126/train0": 1.0726270377635956, + "2054899126/train1": 1.0755530372262, + "2054899126/train2": 1.0683744624257088, + "2061635888/train0": 0.07642464339733124, + "2061635888/train1": 0.09046469628810883, + "2061635888/train2": 0.07902413606643677, + "211245842/train0": 0.5822281837463379, + "211245842/train1": 0.2016763538122177, + "211245842/train2": 0.2142850160598755, + "278008524/train0": 1.0722030997276306, + "278008524/train1": 1.1246033310890198, + "278008524/train2": 1.157032236456871, + "307068879/train0": 0.20280277729034424, + "307068879/train1": 0.20222051441669464, + "307068879/train2": 0.2035302072763443, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 0.8275061845779419, + "440577711/train0": 0.2057151049375534, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.203963503241539, + "441046872/train1": 0.20609354972839355, + "441046872/train2": 0.2042645812034607, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 0.16448359191417694, + "585904130/train1": 0.15698926150798798, + "585904130/train2": 0.2938849627971649, + "608016290/train0": 0.6124615669250488, + "608016290/train1": 0.6101007461547852, + "608016290/train2": 0.5705108642578125, + "626486813/train0": 0.05774742551147938, + "626486813/train1": 0.0619367565959692, + "626486813/train2": 0.08006523177027702, + "636680137/train0": 0.10778365284204483, + "636680137/train1": 0.11253269761800766, + "636680137/train2": 0.1247115358710289, + "655884605/train0": 0.15739890933036804, + "655884605/train1": 0.15038728713989258, + "655884605/train2": 0.17242512106895447, + "672328262/train0": 0.05979333445429802, + "672328262/train1": 0.053388457745313644, + "672328262/train2": 0.07216622680425644, + "701667486/train0": 0.7879213094711304, + "701667486/train1": 0.7947455048561096, + "701667486/train2": 2.0, + "747260437/train0": 0.19230616092681885, + "747260437/train1": 0.1342230886220932, + "747260437/train2": 0.18527252972126007, + "750448296/train0": 0.20943730572859445, + "750448296/train1": 0.22532296180725098, + "750448296/train2": 0.2481476068496704, + "767747424/train0": 0.5789013803005219, + "767747424/train1": 0.5825987188145518, + "767747424/train2": 0.5809952197596431 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.021850009759267, + "low": 0.5703265610120514, + "mean": 0.7913498574552512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7913498574552512, + "per_task": { + "LiftPegUpright-v1": 0.6601470321416855, + "PickCube-v1": 1.021850009759267, + "PullCube-v1": 0.46170220340291657, + "PushCube-v1": 1.1790093589160178, + "StackCube-v1": 0.634040683056369 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 0.7532538059684966, + "low": 0.5049100180880891, + "mean": 0.6232009150553495, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 4.259027777777778, + "low": 3.436111111111111, + "mean": 3.8520833333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.8520833333333333, + "per_seed": { + "1073161778/train0": 7.0, + "1073161778/train1": 2.0, + "1073161778/train2": 6.0, + "1113371261/train0": 0.0, + "1113371261/train1": 7.0, + "1113371261/train2": 6.0, + "1151410346/train0": 0.0, + "1151410346/train1": 3.0, + "1151410346/train2": 7.0, + "1261930456/train0": 2.0, + "1261930456/train1": 3.0, + "1261930456/train2": 6.0, + "1324133122/train0": 4.0, + "1324133122/train1": 4.0, + "1324133122/train2": 2.0, + "1324642411/train0": 7.0, + "1324642411/train1": 3.0, + "1324642411/train2": 3.0, + "1375492786/train0": 6.0, + "1375492786/train1": 4.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 6.0, + "141984615/train2": 6.0, + "1461069713/train0": 1.0, + "1461069713/train1": 4.0, + "1461069713/train2": 7.0, + "1480856810/train0": 6.0, + "1480856810/train1": 0.0, + "1480856810/train2": 2.0, + "1509980664/train0": 5.0, + "1509980664/train1": 1.0, + "1509980664/train2": 7.0, + "1522272686/train0": 6.0, + "1522272686/train1": 7.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 4.0, + "1578990937/train0": 4.0, + "1578990937/train1": 4.0, + "1578990937/train2": 5.0, + "1747178030/train0": 7.0, + "1747178030/train1": 6.0, + "1747178030/train2": 5.0, + "1790188787/train0": 0.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 1.0, + "1822210471/train1": 5.0, + "1822210471/train2": 7.0, + "1845489013/train0": 0.0, + "1845489013/train1": 6.0, + "1845489013/train2": 1.0, + "1999022574/train0": 7.0, + "1999022574/train1": 5.0, + "1999022574/train2": 3.0, + "2012211287/train0": 7.0, + "2012211287/train1": 0.0, + "2012211287/train2": 7.0, + "2038594191/train0": 4.0, + "2038594191/train1": 0.0, + "2038594191/train2": 2.0, + "2054899126/train0": 4.5, + "2054899126/train1": 3.0, + "2054899126/train2": 1.0, + "2061635888/train0": 4.0, + "2061635888/train1": 4.0, + "2061635888/train2": 2.0, + "211245842/train0": 5.0, + "211245842/train1": 3.0, + "211245842/train2": 1.0, + "278008524/train0": 4.5, + "278008524/train1": 1.0, + "278008524/train2": 3.5, + "307068879/train0": 6.0, + "307068879/train1": 1.0, + "307068879/train2": 6.0, + "412102563/train0": 6.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 7.0, + "440577711/train1": 7.0, + "440577711/train2": 1.0, + "441046872/train0": 7.0, + "441046872/train1": 3.0, + "441046872/train2": 6.0, + "509430392/train0": 7.0, + "509430392/train1": 5.0, + "509430392/train2": 3.0, + "585904130/train0": 6.0, + "585904130/train1": 7.0, + "585904130/train2": 1.0, + "608016290/train0": 7.0, + "608016290/train1": 1.0, + "608016290/train2": 6.0, + "626486813/train0": 4.0, + "626486813/train1": 1.5, + "626486813/train2": 2.5, + "636680137/train0": 4.0, + "636680137/train1": 3.0, + "636680137/train2": 7.0, + "655884605/train0": 3.0, + "655884605/train1": 3.0, + "655884605/train2": 6.0, + "672328262/train0": 2.0, + "672328262/train1": 7.0, + "672328262/train2": 5.0, + "701667486/train0": 7.0, + "701667486/train1": 6.0, + "701667486/train2": 3.0, + "747260437/train0": 6.0, + "747260437/train1": 0.0, + "747260437/train2": 2.0, + "750448296/train0": 2.3333333333333335, + "750448296/train1": 4.0, + "750448296/train2": 2.6666666666666665, + "767747424/train0": 4.0, + "767747424/train1": 2.25, + "767747424/train2": 4.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 4.713939393939394, + "low": 3.6152727272727274, + "mean": 4.164606060606061, + "n": 5, + "num_samples": 1000 + }, + "mean": 4.164606060606061, + "per_task": { + "LiftPegUpright-v1": 4.8, + "PickCube-v1": 5.0, + "PullCube-v1": 3.3866666666666667, + "PushCube-v1": 3.6666666666666665, + "StackCube-v1": 3.9696969696969697 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 4.090277777777778, + "low": 3.3472222222222223, + "mean": 3.736111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.37430555555555556, + "low": 0.21319444444444444, + "mean": 0.2888888888888889, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.2888888888888889, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 0.5, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7946666666666667, + "low": 0.08557575757575757, + "mean": 0.44012121212121214, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.44012121212121214, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.30666666666666664, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.06060606060606061 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2152777777777778, + "mean": 0.2986111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.004166666666666667, + "low": -0.04375, + "mean": -0.01944444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.01944444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": -0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": -1.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": -0.3333333333333333, + "750448296/train2": 0.0, + "767747424/train0": -0.25, + "767747424/train1": 0.0, + "767747424/train2": -0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.03806060606060606, + "mean": -0.01672727272727273, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.01672727272727273, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.05333333333333334, + "PushCube-v1": 0.0, + "StackCube-v1": -0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": -0.006944444444444444, + "low": -0.06944444444444445, + "mean": -0.034722222222222224, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9057064777434183, + "low": 0.6092873564507397, + "mean": 0.7526983081350206, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7526983081350206, + "per_seed": { + "1073161778/train0": 0.7180102467536926, + "1073161778/train1": 0.7112008929252625, + "1073161778/train2": 0.7205738425254822, + "1113371261/train0": 0.0680118203163147, + "1113371261/train1": 0.09622243046760559, + "1113371261/train2": 0.09743154048919678, + "1151410346/train0": 0.22687169909477234, + "1151410346/train1": 0.2249394953250885, + "1151410346/train2": 0.24776706099510193, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.05331718921661377, + "1324133122/train1": 0.0843733698129654, + "1324133122/train2": 0.0906635969877243, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.01308689545840025, + "141984615/train1": 0.03492546081542969, + "141984615/train2": 0.03257067874073982, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.05644957348704338, + "1461069713/train2": 0.11542954295873642, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.21791090071201324, + "1549324805/train1": 0.3088321089744568, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21093030273914337, + "1578990937/train1": 0.21256697177886963, + "1578990937/train2": 0.20704451203346252, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.035609763115644455, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.04166736081242561, + "1845489013/train0": 0.035594623535871506, + "1845489013/train1": 0.06991276890039444, + "1845489013/train2": 0.07725050300359726, + "1999022574/train0": 0.5931925177574158, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7166993021965027, + "2012211287/train1": 0.719451367855072, + "2012211287/train2": 0.721496045589447, + "2038594191/train0": 0.08836108446121216, + "2038594191/train1": 0.15308313071727753, + "2038594191/train2": 0.10970946401357651, + "2054899126/train0": 2.0, + "2054899126/train1": 1.0613084435462952, + "2054899126/train2": 2.0, + "2061635888/train0": 0.07642464339733124, + "2061635888/train1": 0.08908586204051971, + "2061635888/train2": 0.07194386422634125, + "211245842/train0": 0.16008998453617096, + "211245842/train1": 0.17465433478355408, + "211245842/train2": 0.18301993608474731, + "278008524/train0": 1.0658646821975708, + "278008524/train1": 1.1246033310890198, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.19984664022922516, + "307068879/train1": 0.20222051441669464, + "307068879/train2": 0.20032821595668793, + "412102563/train0": 0.15669304132461548, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.203963503241539, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2042645812034607, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.14155812561511993, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.13704223930835724, + "608016290/train0": 0.529986560344696, + "608016290/train1": 0.5021255612373352, + "608016290/train2": 0.38905441761016846, + "626486813/train0": 0.03713102079927921, + "626486813/train1": 0.05393064022064209, + "626486813/train2": 0.06953043676912785, + "636680137/train0": 0.04850691556930542, + "636680137/train1": 0.07379839569330215, + "636680137/train2": 0.08920355886220932, + "655884605/train0": 0.14302301406860352, + "655884605/train1": 0.12636350095272064, + "655884605/train2": 0.1574591100215912, + "672328262/train0": 0.045009810477495193, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.0718381404876709, + "701667486/train0": 0.6136738657951355, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7211182241638502, + "750448296/train1": 0.1491819222768148, + "750448296/train2": 0.8722578237454096, + "767747424/train0": 0.5703066997230053, + "767747424/train1": 1.056986487004906, + "767747424/train2": 0.5751176299527287 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6273443445563316, + "low": 0.49430630508945744, + "mean": 1.0508867578910608, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.0508867578910608, + "per_task": { + "LiftPegUpright-v1": 0.46326595544815063, + "PickCube-v1": 2.0, + "PullCube-v1": 0.6904245187963048, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.4272875478773406 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8848815423876254, + "low": 0.6088691439363174, + "mean": 0.7566216745375035, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.018449938262347133, + "low": -0.09586112364324637, + "mean": -0.05010516805082767, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.05010516805082767, + "per_seed": { + "1073161778/train0": -0.007507741451263428, + "1073161778/train1": -0.014317095279693604, + "1073161778/train2": -0.004944145679473877, + "1113371261/train0": -0.043087467551231384, + "1113371261/train1": -0.01487685739994049, + "1113371261/train2": -0.013667747378349304, + "1151410346/train0": -0.0016018152236938477, + "1151410346/train1": -0.0035340189933776855, + "1151410346/train2": 0.019293546676635742, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": -0.030909866094589233, + "1324133122/train1": 0.00014631450176239014, + "1324133122/train2": 0.006436541676521301, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.1547458367422223, + "141984615/train1": -0.13290727138519287, + "141984615/train2": -0.13526205345988274, + "1461069713/train0": 0.0, + "1461069713/train1": -0.17207801714539528, + "1461069713/train2": -0.11309804767370224, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.04541558027267456, + "1549324805/train1": 0.1363367885351181, + "1549324805/train2": 0.0, + "1578990937/train0": -0.0033356696367263794, + "1578990937/train1": -0.001699000597000122, + "1578990937/train2": -0.0072214603424072266, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": -0.05881569907069206, + "1822210471/train1": 0.0, + "1822210471/train2": -0.052758101373910904, + "1845489013/train0": -0.0769633837044239, + "1845489013/train1": -0.04264523833990097, + "1845489013/train2": -0.03530750423669815, + "1999022574/train0": -0.23477637767791748, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": -0.011547386646270752, + "2012211287/train1": -0.008795320987701416, + "2012211287/train2": -0.006750643253326416, + "2038594191/train0": -0.006679512560367584, + "2038594191/train1": 0.058042533695697784, + "2038594191/train2": 0.014668866991996765, + "2054899126/train0": 0.0, + "2054899126/train1": -0.9386915564537048, + "2054899126/train2": 0.0, + "2061635888/train0": -0.009782284498214722, + "2061635888/train1": 0.002878934144973755, + "2061635888/train2": -0.014263063669204712, + "211245842/train0": -0.02421446144580841, + "211245842/train1": -0.009650111198425293, + "211245842/train2": -0.0012845098972320557, + "278008524/train0": -0.05914252996444702, + "278008524/train1": -0.0004038810729980469, + "278008524/train2": 0.0, + "307068879/train0": -0.009305715560913086, + "307068879/train1": -0.0069318413734436035, + "307068879/train2": -0.008824139833450317, + "412102563/train0": -1.8433069586753845, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": -0.005804136395454407, + "441046872/train1": 0.0, + "441046872/train2": -0.005503058433532715, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.07052158564329147, + "585904130/train1": 0.0, + "585904130/train2": 0.06600569933652878, + "608016290/train0": 0.004626452922821045, + "608016290/train1": -0.023234546184539795, + "608016290/train2": -0.13630568981170654, + "626486813/train0": -0.00460987351834774, + "626486813/train1": 0.012189745903015137, + "626486813/train2": 0.027789542451500893, + "636680137/train0": -0.07106667757034302, + "636680137/train1": -0.04577519744634628, + "636680137/train2": -0.030370034277439117, + "655884605/train0": -0.0010523796081542969, + "655884605/train1": -0.01771189272403717, + "655884605/train2": 0.013383716344833374, + "672328262/train0": -0.060924429446458817, + "672328262/train1": 0.0, + "672328262/train2": -0.03409609943628311, + "701667486/train0": -0.1776002049446106, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": -0.039385102689266205, + "750448296/train1": -0.6113214045763016, + "750448296/train2": 0.11175449689229329, + "767747424/train0": -0.5044020563364029, + "767747424/train1": -0.01772226905450225, + "767747424/train2": -0.49959112610667944 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.0032686001062393185, + "low": -0.08231042620246157, + "mean": -0.040448589019025816, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.040448589019025816, + "per_task": { + "LiftPegUpright-v1": -0.016343000531196594, + "PickCube-v1": 0.0, + "PullCube-v1": -0.1128260932241877, + "PushCube-v1": 0.0, + "StackCube-v1": -0.0730738513397448 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": -0.026475677939338818, + "low": -0.14210378661260215, + "mean": -0.07721207704162225, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.23132771001223268, + "low": 0.07666435047544333, + "mean": 0.14942124262064074, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.14942124262064074, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0022280216217041016, + "1073161778/train2": 1.2794261574745178, + "1113371261/train0": 0.0060281902551651, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0029332488775253296, + "1151410346/train0": 0.34162279963493347, + "1151410346/train1": 0.01898777484893799, + "1151410346/train2": 0.31524333357810974, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.009534716606140137, + "1324133122/train1": 0.0, + "1324133122/train2": 0.007487863302230835, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.010632594116032124, + "141984615/train1": 0.009052593261003494, + "141984615/train2": 0.011906486004590988, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.167920172214508, + "1509980664/train1": 1.167920172214508, + "1509980664/train2": 0.006812572479248047, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.04154062271118164, + "1578990937/train0": 0.003326788544654846, + "1578990937/train1": 0.00506339967250824, + "1578990937/train2": 0.025547489523887634, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0024167299270629883, + "1822210471/train1": 0.0, + "1822210471/train2": 0.017152011394500732, + "1845489013/train0": 0.007289845496416092, + "1845489013/train1": 0.025380156934261322, + "1845489013/train2": 0.008944690227508545, + "1999022574/train0": 0.0038532018661499023, + "1999022574/train1": 1.1720311045646667, + "1999022574/train2": 0.0, + "2012211287/train0": 0.011076092720031738, + "2012211287/train1": 1.280548632144928, + "2012211287/train2": 0.00554656982421875, + "2038594191/train0": 0.06411068141460419, + "2038594191/train1": 0.03719758987426758, + "2038594191/train2": 0.1390988454222679, + "2054899126/train0": 0.0, + "2054899126/train1": 0.014244593679904938, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0013788342475891113, + "2061635888/train2": 0.00708027184009552, + "211245842/train0": 0.42213819921016693, + "211245842/train1": 0.027022019028663635, + "211245842/train2": 0.031265079975128174, + "278008524/train0": 0.0063384175300598145, + "278008524/train1": 0.0, + "278008524/train2": 0.03202502429485321, + "307068879/train0": 0.0029561370611190796, + "307068879/train1": 0.0, + "307068879/train2": 0.003201991319656372, + "412102563/train0": 1.8433069586753845, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.9592653959989548, + "509430392/train1": 1.9592653959989548, + "509430392/train2": 1.9592653959989548, + "585904130/train0": 0.022925466299057007, + "585904130/train1": 0.08595272153615952, + "585904130/train2": 0.15684272348880768, + "608016290/train0": 0.08247500658035278, + "608016290/train1": 0.10797518491744995, + "608016290/train2": 0.18145644664764404, + "626486813/train0": 0.020616404712200165, + "626486813/train1": 0.00800611637532711, + "626486813/train2": 0.010534795001149178, + "636680137/train0": 0.05927673727273941, + "636680137/train1": 0.038734301924705505, + "636680137/train2": 0.03550797700881958, + "655884605/train0": 0.014375895261764526, + "655884605/train1": 0.024023786187171936, + "655884605/train2": 0.014966011047363281, + "672328262/train0": 0.014783523976802826, + "672328262/train1": 0.0, + "672328262/train2": 0.00032808631658554077, + "701667486/train0": 0.17424744367599487, + "701667486/train1": 0.0034714341163635254, + "701667486/train2": 1.208725929260254, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.07583838452895482, + "750448296/train1": 0.0761410395304362, + "750448296/train2": 0.0, + "767747424/train0": 0.008594680577516556, + "767747424/train1": 0.022256564814597368, + "767747424/train2": 0.0058775898069143295 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2673315591884382, + "low": 0.04782764960825443, + "mean": 0.1575796043983463, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.1575796043983463, + "per_task": { + "LiftPegUpright-v1": 0.19712601602077484, + "PickCube-v1": 0.0, + "PullCube-v1": 0.02100611601024866, + "PushCube-v1": 0.32654423266649246, + "StackCube-v1": 0.24322165729421558 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.20277719939541486, + "low": 0.07080135096718247, + "mean": 0.12803097097073785, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14166666666666666, + "low": 0.03333333333333333, + "mean": 0.08333333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.08333333333333333, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.15393939393939393, + "low": 0.026666666666666665, + "mean": 0.0903030303030303, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.0903030303030303, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.15151515151515152 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.11805555555555555, + "low": 0.034722222222222224, + "mean": 0.06944444444444445, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6597222222222222, + "low": 0.4847222222222222, + "mean": 0.5708333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5708333333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8072727272727273, + "low": 0.40533333333333327, + "mean": 0.6063030303030303, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6063030303030303, + "per_task": { + "LiftPegUpright-v1": 0.8666666666666667, + "PickCube-v1": 0.6666666666666666, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.8181818181818182 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.5972222222222222, + "low": 0.4375, + "mean": 0.5138888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1496533814558965, + "low": 0.8442967605000982, + "mean": 0.9982878092624661, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.9982878092624661, + "per_seed": { + "1073161778/train0": 1.2819897532463074, + "1073161778/train1": 1.2865710854530334, + "1073161778/train2": 0.0, + "1113371261/train0": 1.9259599894285202, + "1113371261/train1": 1.9037775695323944, + "1113371261/train2": 1.899635210633278, + "1151410346/train0": 1.4315055012702942, + "1151410346/train1": 1.7560727298259735, + "1151410346/train2": 1.4369896054267883, + "1261930456/train0": 1.90321846306324, + "1261930456/train1": 1.9000287055969238, + "1261930456/train2": 1.9145455807447433, + "1324133122/train0": 1.937148094177246, + "1324133122/train1": 1.9156266301870346, + "1324133122/train2": 1.9018485397100449, + "1324642411/train0": 1.7899941802024841, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.8525634557008743, + "1375492786/train2": 1.805472657084465, + "141984615/train0": 0.2620577812194824, + "141984615/train1": 0.2417992167174816, + "141984615/train2": 0.24130010604858398, + "1461069713/train0": 0.10317284241318703, + "1461069713/train1": 0.10289287939667702, + "1461069713/train2": 0.04391290992498398, + "1480856810/train0": 0.06770022213459015, + "1480856810/train1": 0.0, + "1480856810/train2": 0.013253256678581238, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.16110759973526, + "1522272686/train0": 1.476680040359497, + "1522272686/train1": 1.4577699303627014, + "1522272686/train2": 0.0, + "1549324805/train0": 0.07954518496990204, + "1549324805/train1": 0.0, + "1549324805/train2": 0.08342014253139496, + "1578990937/train0": 1.7857429087162018, + "1578990937/train1": 1.7823696285486221, + "1578990937/train2": 1.7674079984426498, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8137669563293457, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9619735069572926, + "1822210471/train1": 1.924605906009674, + "1822210471/train2": 1.9411806277930737, + "1845489013/train0": 1.9571155309677124, + "1845489013/train1": 1.9047070741653442, + "1845489013/train2": 1.9138048067688942, + "1999022574/train0": 1.4029542803764343, + "1999022574/train1": 0.0, + "1999022574/train2": 1.2029985189437866, + "2012211287/train0": 1.2722246050834656, + "2012211287/train1": 0.0, + "2012211287/train2": 1.2729573845863342, + "2038594191/train0": 1.8475282341241837, + "2038594191/train1": 1.809719279408455, + "2038594191/train2": 1.7511916905641556, + "2054899126/train0": 0.9273729622364044, + "2054899126/train1": 0.9244469627737999, + "2054899126/train2": 0.9316255375742912, + "2061635888/train0": 1.9235753566026688, + "2061635888/train1": 1.9095353037118912, + "2061635888/train2": 1.9209758639335632, + "211245842/train0": 1.417771816253662, + "211245842/train1": 1.7983236461877823, + "211245842/train2": 1.7857149839401245, + "278008524/train0": 0.04170554131269455, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.7971972227096558, + "307068879/train1": 1.7977794855833054, + "307068879/train2": 1.7964697927236557, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.172493815422058, + "440577711/train0": 1.7942848950624466, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.796036496758461, + "441046872/train1": 1.7939064502716064, + "441046872/train2": 1.7957354187965393, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.835516408085823, + "585904130/train1": 1.843010738492012, + "585904130/train2": 1.706115037202835, + "608016290/train0": 1.3875384330749512, + "608016290/train1": 1.3898992538452148, + "608016290/train2": 1.4294891357421875, + "626486813/train0": 0.1078349556773901, + "626486813/train1": 0.10364562459290028, + "626486813/train2": 0.08551714941859245, + "636680137/train0": 0.06278067082166672, + "636680137/train1": 0.05803162604570389, + "636680137/train2": 0.04585278779268265, + "655884605/train0": 1.842601090669632, + "655884605/train1": 1.8496127128601074, + "655884605/train2": 1.8275748789310455, + "672328262/train0": 1.940206665545702, + "672328262/train1": 1.9466115422546864, + "672328262/train2": 1.9278337731957436, + "701667486/train0": 1.2120786905288696, + "701667486/train1": 1.2052544951438904, + "701667486/train2": 0.0, + "747260437/train0": 0.08635708689689636, + "747260437/train1": 0.144440159201622, + "747260437/train2": 0.09339071810245514, + "750448296/train0": 1.1985591898361843, + "750448296/train1": 1.1875135501225789, + "750448296/train2": 1.1598488887151082, + "767747424/train0": 0.10475751012563705, + "767747424/train1": 0.1008116127923131, + "767747424/train2": 0.09709237236529589 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2779549119689249, + "low": 0.7197053071856498, + "mean": 0.9988301095772872, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9988301095772872, + "per_task": { + "LiftPegUpright-v1": 1.3398529678583144, + "PickCube-v1": 0.9781499902407328, + "PullCube-v1": 0.6881442584097386, + "PushCube-v1": 0.6220440144340197, + "StackCube-v1": 1.365959316943631 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 1.045588894850678, + "low": 0.7773104406013671, + "mean": 0.9091424554741631, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.09621934768667245, + "low": -0.09621934768667245, + "mean": -0.09621934768667245, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.09621934768667245, + "per_seed": { + "1073161778/train0": -0.09621934768667256, + "1073161778/train1": -0.09621934768667256, + "1073161778/train2": -0.09621934768667256, + "1113371261/train0": -0.09621934768667256, + "1113371261/train1": -0.09621934768667256, + "1113371261/train2": -0.09621934768667256, + "1151410346/train0": -0.09621934768667256, + "1151410346/train1": -0.09621934768667256, + "1151410346/train2": -0.09621934768667256, + "1261930456/train0": -0.09621934768667256, + "1261930456/train1": -0.09621934768667256, + "1261930456/train2": -0.09621934768667256, + "1324133122/train0": -0.09621934768667256, + "1324133122/train1": -0.09621934768667256, + "1324133122/train2": -0.09621934768667256, + "1324642411/train0": -0.09621934768667256, + "1324642411/train1": -0.09621934768667256, + "1324642411/train2": -0.09621934768667256, + "1375492786/train0": -0.09621934768667256, + "1375492786/train1": -0.09621934768667256, + "1375492786/train2": -0.09621934768667256, + "141984615/train0": -0.09621934768667256, + "141984615/train1": -0.09621934768667256, + "141984615/train2": -0.09621934768667256, + "1461069713/train0": -0.09621934768667256, + "1461069713/train1": -0.09621934768667256, + "1461069713/train2": -0.09621934768667256, + "1480856810/train0": -0.09621934768667256, + "1480856810/train1": -0.09621934768667256, + "1480856810/train2": -0.09621934768667256, + "1509980664/train0": -0.09621934768667256, + "1509980664/train1": -0.09621934768667256, + "1509980664/train2": -0.09621934768667256, + "1522272686/train0": -0.09621934768667256, + "1522272686/train1": -0.09621934768667256, + "1522272686/train2": -0.09621934768667256, + "1549324805/train0": -0.09621934768667256, + "1549324805/train1": -0.09621934768667256, + "1549324805/train2": -0.09621934768667256, + "1578990937/train0": -0.09621934768667256, + "1578990937/train1": -0.09621934768667256, + "1578990937/train2": -0.09621934768667256, + "1747178030/train0": -0.09621934768667256, + "1747178030/train1": -0.09621934768667256, + "1747178030/train2": -0.09621934768667256, + "1790188787/train0": -0.09621934768667256, + "1790188787/train1": -0.09621934768667256, + "1790188787/train2": -0.09621934768667256, + "1822210471/train0": -0.09621934768667256, + "1822210471/train1": -0.09621934768667256, + "1822210471/train2": -0.09621934768667256, + "1845489013/train0": -0.09621934768667256, + "1845489013/train1": -0.09621934768667256, + "1845489013/train2": -0.09621934768667256, + "1999022574/train0": -0.09621934768667256, + "1999022574/train1": -0.09621934768667256, + "1999022574/train2": -0.09621934768667256, + "2012211287/train0": -0.09621934768667256, + "2012211287/train1": -0.09621934768667256, + "2012211287/train2": -0.09621934768667256, + "2038594191/train0": -0.09621934768667256, + "2038594191/train1": -0.09621934768667256, + "2038594191/train2": -0.09621934768667256, + "2054899126/train0": -0.09621934768667256, + "2054899126/train1": -0.09621934768667256, + "2054899126/train2": -0.09621934768667256, + "2061635888/train0": -0.09621934768667256, + "2061635888/train1": -0.09621934768667256, + "2061635888/train2": -0.09621934768667256, + "211245842/train0": -0.09621934768667256, + "211245842/train1": -0.09621934768667256, + "211245842/train2": -0.09621934768667256, + "278008524/train0": -0.09621934768667256, + "278008524/train1": -0.09621934768667256, + "278008524/train2": -0.09621934768667256, + "307068879/train0": -0.09621934768667256, + "307068879/train1": -0.09621934768667256, + "307068879/train2": -0.09621934768667256, + "412102563/train0": -0.09621934768667256, + "412102563/train1": -0.09621934768667256, + "412102563/train2": -0.09621934768667256, + "440577711/train0": -0.09621934768667256, + "440577711/train1": -0.09621934768667256, + "440577711/train2": -0.09621934768667256, + "441046872/train0": -0.09621934768667256, + "441046872/train1": -0.09621934768667256, + "441046872/train2": -0.09621934768667256, + "509430392/train0": -0.09621934768667256, + "509430392/train1": -0.09621934768667256, + "509430392/train2": -0.09621934768667256, + "585904130/train0": -0.09621934768667256, + "585904130/train1": -0.09621934768667256, + "585904130/train2": -0.09621934768667256, + "608016290/train0": -0.09621934768667256, + "608016290/train1": -0.09621934768667256, + "608016290/train2": -0.09621934768667256, + "626486813/train0": -0.09621934768667256, + "626486813/train1": -0.09621934768667256, + "626486813/train2": -0.09621934768667256, + "636680137/train0": -0.09621934768667256, + "636680137/train1": -0.09621934768667256, + "636680137/train2": -0.09621934768667256, + "655884605/train0": -0.09621934768667256, + "655884605/train1": -0.09621934768667256, + "655884605/train2": -0.09621934768667256, + "672328262/train0": -0.09621934768667256, + "672328262/train1": -0.09621934768667256, + "672328262/train2": -0.09621934768667256, + "701667486/train0": -0.09621934768667256, + "701667486/train1": -0.09621934768667256, + "701667486/train2": -0.09621934768667256, + "747260437/train0": -0.09621934768667256, + "747260437/train1": -0.09621934768667256, + "747260437/train2": -0.09621934768667256, + "750448296/train0": -0.09621934768667256, + "750448296/train1": -0.09621934768667256, + "750448296/train2": -0.09621934768667256, + "767747424/train0": -0.09621934768667256, + "767747424/train1": -0.09621934768667256, + "767747424/train2": -0.09621934768667256 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.09621934768667254, + "low": -0.09621934768667262, + "mean": -0.09621934768667258, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.09621934768667258, + "per_task": { + "LiftPegUpright-v1": -0.09621934768667259, + "PickCube-v1": -0.09621934768667256, + "PullCube-v1": -0.09621934768667265, + "PushCube-v1": -0.09621934768667259, + "StackCube-v1": -0.09621934768667252 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": -0.09621934768667237, + "low": -0.09621934768667237, + "mean": -0.09621934768667237, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": -0.09621934768667256, + "threshold_scope": "global", + "weights": [ + -0.36334951815701455, + 0.008940492241759301, + -0.009148829721047302, + 0.006589680641868476, + -0.00033165978980666255, + -0.00902675571773855, + 0.0298694613791623, + -0.23823528748064504, + 0.1464507847218249, + 1.9905443097388735e-14, + -0.015895985432583082, + 0.006946132668822713, + -0.006713693407178312, + 0.016060999964764733, + -0.0075737098585682006, + 0.010178503760364777, + 0.007954417903086153, + -0.0018073658401327464, + -0.015895985432538673, + 0.006946132668780081, + -0.006713693407121468, + 0.016060999964782496, + -0.007573709858574418, + 0.010178503760414515, + 0.007954417903107469, + -0.0018073658401398518, + -0.01557706418356064, + -0.004014733662080161, + 0.0126908865442239, + 0.004313681841914274, + -0.008196328693484745, + -0.005180791940876617, + 0.012774627618391321, + -0.012864474948852234, + -1.7650439898042923e-15, + 0.019403328915910123, + 0.014109890698144056, + -0.21709199708283156, + 0.00041776855218902953, + -0.29453814191703165, + 0.040979728804137694, + 0.15982263438864486, + 0.07240114825460792, + -0.028505041877231185, + 0.10139036011579161, + 0.18193776096968656, + 0.023251922258227253, + 0.0754488004075558, + 0.1229171931494002, + 0.11654493427519166, + -0.01756365151423256, + -0.1536796249120549, + -0.07717409332673535, + -0.09187144350528459, + 0.017698272307886315, + -0.017467325636531028, + -0.10218523215030831, + -0.10341392859653945, + -0.07805263465075105, + 0.09193502388711905, + 0.08085198441288588, + 0.034168996563514395, + -0.0582430378575729, + -0.09138365596105104, + 0.05711296580434927, + -0.19235681715252917, + -0.03981506356573501, + -0.03057544132885276, + 0.03622647372207943, + -0.10334532763088866, + 0.10571841928367814, + 0.1154413468096202, + -0.013416211903966302, + -0.007757979908048551, + 0.17728980221287727, + 0.03757390744884829, + 0.10628767988744098, + 0.05857202675823944, + -0.17007761910477048 + ] +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..7a455cb1a4407daba8a00cad67d8f8146d6e0d58 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics_by_seed.json @@ -0,0 +1,3234 @@ +{ + "1073161778/train0": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.515023875986096, + "pairwise_causal_calibration_ece": 0.09211898115676109, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11698321977507692, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7180102467536926, + "selected_utility_gain_over_base": -0.007507741451263428, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "tau": -0.09621934768667256 + }, + "1073161778/train1": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5164447494303727, + "pairwise_causal_calibration_ece": 0.019269536283913036, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.024067035012088067, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7112008929252625, + "selected_utility_gain_over_base": -0.014317095279693604, + "selector_regret": 0.0022280216217041016, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "tau": -0.09621934768667256 + }, + "1073161778/train2": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5168209602215624, + "pairwise_causal_calibration_ece": 0.19539238879299098, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.005081361506452997, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7205738425254822, + "selected_utility_gain_over_base": -0.004944145679473877, + "selector_regret": 1.2794261574745178, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5335033957491659, + "pairwise_causal_calibration_ece": 0.03350339574916594, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11300812099822777, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0680118203163147, + "selected_utility_gain_over_base": -0.043087467551231384, + "selector_regret": 0.0060281902551651, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "tau": -0.09621934768667256 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5290117220540342, + "pairwise_causal_calibration_ece": 0.10044029348260569, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12608184102453235, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09622243046760559, + "selected_utility_gain_over_base": -0.01487685739994049, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "tau": -0.09621934768667256 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5303568147094659, + "pairwise_causal_calibration_ece": 0.14821461386196266, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16044763104706833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09743154048919678, + "selected_utility_gain_over_base": -0.013667747378349304, + "selector_regret": 0.0029332488775253296, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "tau": -0.09621934768667256 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5219633914019057, + "pairwise_causal_calibration_ece": 0.04946518002666567, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22732175807375893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22687169909477234, + "selected_utility_gain_over_base": -0.0016018152236938477, + "selector_regret": 0.34162279963493347, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "tau": -0.09621934768667256 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5191499717816545, + "pairwise_causal_calibration_ece": 0.1620071146387973, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10383730061951357, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2249394953250885, + "selected_utility_gain_over_base": -0.0035340189933776855, + "selector_regret": 0.01898777484893799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "tau": -0.09621934768667256 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5294358690784804, + "pairwise_causal_calibration_ece": 0.3634212737786625, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.26548608597987267, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.24776706099510193, + "selected_utility_gain_over_base": 0.019293546676635742, + "selector_regret": 0.31524333357810974, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "tau": -0.09621934768667256 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5103233692062079, + "pairwise_causal_calibration_ece": 0.09681948793664918, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.4662863957877161, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "tau": -0.09621934768667256 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5262706089236061, + "pairwise_causal_calibration_ece": 0.06198489463789181, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.4420682590576823, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "tau": -0.09621934768667256 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5549163271210246, + "pairwise_causal_calibration_ece": 0.16205918426388172, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.2751386776848153, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "tau": -0.09621934768667256 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5398428866463935, + "pairwise_causal_calibration_ece": 0.03158568478217802, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21239530032578002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05331718921661377, + "selected_utility_gain_over_base": -0.030909866094589233, + "selector_regret": 0.009534716606140137, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "tau": -0.09621934768667256 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5205020188672198, + "pairwise_causal_calibration_ece": 0.015212266847065892, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21539940459619855, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0843733698129654, + "selected_utility_gain_over_base": 0.00014631450176239014, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "tau": -0.09621934768667256 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5163907247856556, + "pairwise_causal_calibration_ece": 0.05503784664291578, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06331113876354011, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0906635969877243, + "selected_utility_gain_over_base": 0.006436541676521301, + "selector_regret": 0.007487863302230835, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "tau": -0.09621934768667256 + }, + "1324642411/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5780381556995099, + "pairwise_causal_calibration_ece": 0.19552213561759554, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.3335400221980923, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "tau": -0.09621934768667256 + }, + "1324642411/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25925925925925924, + "pairwise_causal_calibration_confidence": 0.5689072335846573, + "pairwise_causal_calibration_ece": 0.3096479743253979, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": -0.28540676920232155, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1324642411/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5520567764502703, + "pairwise_causal_calibration_ece": 0.08422908277413624, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": -0.5068909378194302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 1.0, + "pairwise_causal_calibration_confidence": 0.5999128326661309, + "pairwise_causal_calibration_ece": 0.4000871673338691, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.9897121889793562, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5513845519551138, + "pairwise_causal_calibration_ece": 0.2343297337591719, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.5584765371065843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "tau": -0.09621934768667256 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5465035011084322, + "pairwise_causal_calibration_ece": 0.23921078460585363, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.9167380947098389, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "tau": -0.09621934768667256 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5578282367528864, + "pairwise_causal_calibration_ece": 0.08947433730730506, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4190457481175329, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01308689545840025, + "selected_utility_gain_over_base": -0.1547458367422223, + "selector_regret": 0.010632594116032124, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "tau": -0.09621934768667256 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5379283824128914, + "pairwise_causal_calibration_ece": 0.18078552527003428, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15347700374980155, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03492546081542969, + "selected_utility_gain_over_base": -0.13290727138519287, + "selector_regret": 0.009052593261003494, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "tau": -0.09621934768667256 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5626061019037739, + "pairwise_causal_calibration_ece": 0.4197489590466311, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.41265012485850616, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03257067874073982, + "selected_utility_gain_over_base": -0.13526205345988274, + "selector_regret": 0.011906486004590988, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "tau": -0.09621934768667256 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5429040222614606, + "pairwise_causal_calibration_ece": 0.2070959777385393, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2893398512416173, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "tau": -0.09621934768667256 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5353090444202274, + "pairwise_causal_calibration_ece": 0.14326238415120118, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.26002782677736214, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05644957348704338, + "selected_utility_gain_over_base": -0.17207801714539528, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "tau": -0.09621934768667256 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5652627006295361, + "pairwise_causal_calibration_ece": 0.17299321545306823, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.866998389422337, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11542954295873642, + "selected_utility_gain_over_base": -0.11309804767370224, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "tau": -0.09621934768667256 + }, + "1480856810/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5347267964365477, + "pairwise_causal_calibration_ece": 0.2132982250079763, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.46015939778469717, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "tau": -0.09621934768667256 + }, + "1480856810/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5393866728393286, + "pairwise_causal_calibration_ece": 0.07510095855361425, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.775305326035764, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1480856810/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5353643069980405, + "pairwise_causal_calibration_ece": 0.0717785501448166, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.7567849205182233, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "tau": -0.09621934768667256 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.28, + "pairwise_causal_calibration_confidence": 0.5190763569223411, + "pairwise_causal_calibration_ece": 0.23907635692234108, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.189222442683031, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.72, + "pairwise_causal_calibration_confidence": 0.5170671799261125, + "pairwise_causal_calibration_ece": 0.20293282007388747, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.2764527397331097, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5137167555753096, + "pairwise_causal_calibration_ece": 0.021997530138976096, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.36227529869707314, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "tau": -0.09621934768667256 + }, + "1522272686/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.6068801363670007, + "pairwise_causal_calibration_ece": 0.17883414934728506, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2574720584344846, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "tau": -0.09621934768667256 + }, + "1522272686/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5357279663023194, + "pairwise_causal_calibration_ece": 1.368058803374872e-05, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.8824130953826943, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "tau": -0.09621934768667256 + }, + "1522272686/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5218897792626456, + "pairwise_causal_calibration_ece": 0.22811022073735443, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.5831727753564718, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.6011992431167774, + "pairwise_causal_calibration_ece": 0.18451504259750837, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4839704193432687, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21791090071201324, + "selected_utility_gain_over_base": 0.04541558027267456, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "tau": -0.09621934768667256 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.6130514300483924, + "pairwise_causal_calibration_ece": 0.31551999852303597, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6249019311477833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.3088321089744568, + "selected_utility_gain_over_base": 0.1363367885351181, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5595699588623118, + "pairwise_causal_calibration_ece": 0.05956995886231174, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1195884497983013, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04154062271118164, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "tau": -0.09621934768667256 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5309964278670222, + "pairwise_causal_calibration_ece": 0.21900357213297783, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3729341213514183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21093030273914337, + "selected_utility_gain_over_base": -0.0033356696367263794, + "selector_regret": 0.003326788544654846, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "tau": -0.09621934768667256 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9285714285714286, + "pairwise_causal_calibration_confidence": 0.5395331577195327, + "pairwise_causal_calibration_ece": 0.3890382708518959, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30002172815923867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21256697177886963, + "selected_utility_gain_over_base": -0.001699000597000122, + "selector_regret": 0.00506339967250824, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "tau": -0.09621934768667256 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5467275958451122, + "pairwise_causal_calibration_ece": 0.07656675457436006, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30083926520305015, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20704451203346252, + "selected_utility_gain_over_base": -0.0072214603424072266, + "selector_regret": 0.025547489523887634, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "tau": -0.09621934768667256 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.46805085413062614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.050885567292246775, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.21223227940792033, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1790188787/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5453049027370976, + "pairwise_causal_calibration_ece": 0.13844693675904163, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.2520875088855983, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1790188787/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5203995406793089, + "pairwise_causal_calibration_ece": 0.15817188789211967, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -1.1676288743355996, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "tau": -0.09621934768667256 + }, + "1790188787/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.5253056653691316, + "pairwise_causal_calibration_ece": 0.06560342553995935, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.1440144908506833, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5302641729509556, + "pairwise_causal_calibration_ece": 0.18402154133475868, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0849388442130512, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035609763115644455, + "selected_utility_gain_over_base": -0.05881569907069206, + "selector_regret": 0.0024167299270629883, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "tau": -0.09621934768667256 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5270692288887334, + "pairwise_causal_calibration_ece": 0.09849780031730487, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.12085404649382042, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "tau": -0.09621934768667256 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5271103602141303, + "pairwise_causal_calibration_ece": 0.4199675030712731, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.015212697452195612, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04166736081242561, + "selected_utility_gain_over_base": -0.052758101373910904, + "selector_regret": 0.017152011394500732, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "tau": -0.09621934768667256 + }, + "1845489013/train0": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5100440403448724, + "pairwise_causal_calibration_ece": 0.13281310251227052, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25691250884199157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035594623535871506, + "selected_utility_gain_over_base": -0.0769633837044239, + "selector_regret": 0.007289845496416092, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "tau": -0.09621934768667256 + }, + "1845489013/train1": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5186589916137023, + "pairwise_causal_calibration_ece": 0.23294470589941663, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10389427792008576, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06991276890039444, + "selected_utility_gain_over_base": -0.04264523833990097, + "selector_regret": 0.025380156934261322, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "tau": -0.09621934768667256 + }, + "1845489013/train2": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5135058626142434, + "pairwise_causal_calibration_ece": 0.049220148328529056, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.058128320354532526, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07725050300359726, + "selected_utility_gain_over_base": -0.03530750423669815, + "selector_regret": 0.008944690227508545, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "tau": -0.09621934768667256 + }, + "1999022574/train0": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5428168971264415, + "pairwise_causal_calibration_ece": 0.007102611412155846, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06187932823477338, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5931925177574158, + "selected_utility_gain_over_base": -0.23477637767791748, + "selector_regret": 0.0038532018661499023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "tau": -0.09621934768667256 + }, + "1999022574/train1": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5236071274921709, + "pairwise_causal_calibration_ece": 0.23789284177788517, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.2099363129546052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "1999022574/train2": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5379278533729487, + "pairwise_causal_calibration_ece": 0.2835007180556227, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1638733757665311, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "tau": -0.09621934768667256 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5193384170444694, + "pairwise_causal_calibration_ece": 0.3764812741873265, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14968836800979451, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7166993021965027, + "selected_utility_gain_over_base": -0.011547386646270752, + "selector_regret": 0.011076092720031738, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "tau": -0.09621934768667256 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48148148148148145, + "pairwise_causal_calibration_confidence": 0.5195673663835161, + "pairwise_causal_calibration_ece": 0.03808588490203463, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.03517960107744428, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.719451367855072, + "selected_utility_gain_over_base": -0.008795320987701416, + "selector_regret": 1.280548632144928, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5207899434564643, + "pairwise_causal_calibration_ece": 0.34221851488503574, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0952851317844515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.721496045589447, + "selected_utility_gain_over_base": -0.006750643253326416, + "selector_regret": 0.00554656982421875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "tau": -0.09621934768667256 + }, + "2038594191/train0": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5628942850824857, + "pairwise_causal_calibration_ece": 0.061086876087235106, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12575119573418914, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08836108446121216, + "selected_utility_gain_over_base": -0.006679512560367584, + "selector_regret": 0.06411068141460419, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "tau": -0.09621934768667256 + }, + "2038594191/train1": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5410364066615194, + "pairwise_causal_calibration_ece": 0.21960783523294797, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06608733752201157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15308313071727753, + "selected_utility_gain_over_base": 0.058042533695697784, + "selector_regret": 0.03719758987426758, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "tau": -0.09621934768667256 + }, + "2038594191/train2": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5590374392784875, + "pairwise_causal_calibration_ece": 0.05880559485727592, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.26386124774663755, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10970946401357651, + "selected_utility_gain_over_base": 0.014668866991996765, + "selector_regret": 0.1390988454222679, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "tau": -0.09621934768667256 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5729749711313679, + "pairwise_causal_calibration_ece": 0.05621631366113361, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.028673644731745662, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0726270377635956, + "selected_candidate_index": 4.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9273729622364044, + "tau": -0.09621934768667256 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5546822433674421, + "pairwise_causal_calibration_ece": 0.0987696931122925, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.15591689454571314, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0755530372262, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0613084435462952, + "selected_utility_gain_over_base": -0.9386915564537048, + "selector_regret": 0.014244593679904938, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9244469627737999, + "tau": -0.09621934768667256 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5766922131739299, + "pairwise_causal_calibration_ece": 0.06879675055589206, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": -0.14059305249407972, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0683744624257088, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9316255375742912, + "tau": -0.09621934768667256 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5302836142133847, + "pairwise_causal_calibration_ece": 0.29114495721518674, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028349483019357136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07642464339733124, + "selected_utility_gain_over_base": -0.009782284498214722, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "tau": -0.09621934768667256 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5408950551165522, + "pairwise_causal_calibration_ece": 0.17339065916916208, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21267963182785404, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08908586204051971, + "selected_utility_gain_over_base": 0.002878934144973755, + "selector_regret": 0.0013788342475891113, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "tau": -0.09621934768667256 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5213600808814732, + "pairwise_causal_calibration_ece": 0.08578277626138386, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.10091080743716707, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07194386422634125, + "selected_utility_gain_over_base": -0.014263063669204712, + "selector_regret": 0.00708027184009552, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "tau": -0.09621934768667256 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5206119271327049, + "pairwise_causal_calibration_ece": 0.05081664429586652, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17393811200736525, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16008998453617096, + "selected_utility_gain_over_base": -0.02421446144580841, + "selector_regret": 0.42213819921016693, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "tau": -0.09621934768667256 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5442178751645994, + "pairwise_causal_calibration_ece": 0.17006783912111498, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.16983155866909005, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17465433478355408, + "selected_utility_gain_over_base": -0.009650111198425293, + "selector_regret": 0.027022019028663635, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "tau": -0.09621934768667256 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5332794039871745, + "pairwise_causal_calibration_ece": 0.18100631029853975, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09204961172655701, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18301993608474731, + "selected_utility_gain_over_base": -0.0012845098972320557, + "selector_regret": 0.031265079975128174, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "tau": -0.09621934768667256 + }, + "278008524/train0": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5670465168048365, + "pairwise_causal_calibration_ece": 0.020048170169010344, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.43269538414303643, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0722030997276306, + "selected_candidate_index": 4.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0658646821975708, + "selected_utility_gain_over_base": -0.05914252996444702, + "selector_regret": 0.0063384175300598145, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04170554131269455, + "tau": -0.09621934768667256 + }, + "278008524/train1": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5654063126919278, + "pairwise_causal_calibration_ece": 0.05658255793975048, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.35878896805617033, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1246033310890198, + "selected_candidate_index": 1.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1246033310890198, + "selected_utility_gain_over_base": -0.0004038810729980469, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "278008524/train2": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5383303226297562, + "pairwise_causal_calibration_ece": 0.03440626725655044, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.15771232576630806, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.157032236456871, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03202502429485321, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.532810869430912, + "pairwise_causal_calibration_ece": 0.06852515514519775, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.053124156513879084, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19984664022922516, + "selected_utility_gain_over_base": -0.009305715560913086, + "selector_regret": 0.0029561370611190796, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "tau": -0.09621934768667256 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5310502088948758, + "pairwise_causal_calibration_ece": 0.11180693396226715, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0009841947897497316, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20222051441669464, + "selected_utility_gain_over_base": -0.0069318413734436035, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "tau": -0.09621934768667256 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5426502518804291, + "pairwise_causal_calibration_ece": 0.042650251880429216, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0658010800556802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20032821595668793, + "selected_utility_gain_over_base": -0.008824139833450317, + "selector_regret": 0.003201991319656372, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "tau": -0.09621934768667256 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5329987081920804, + "pairwise_causal_calibration_ece": 0.14014156533493755, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03891331669299969, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.15669304132461548, + "selected_utility_gain_over_base": -1.8433069586753845, + "selector_regret": 1.8433069586753845, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.518827017198031, + "pairwise_causal_calibration_ece": 0.054541302912316736, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.21560261538347625, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5238928346234378, + "pairwise_causal_calibration_ece": 0.011821451090847868, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.22361307170053446, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "tau": -0.09621934768667256 + }, + "440577711/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5325688201837939, + "pairwise_causal_calibration_ece": 0.038859751244777474, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.7148050601008141, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "tau": -0.09621934768667256 + }, + "440577711/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07692307692307693, + "pairwise_causal_calibration_confidence": 0.5300053688111541, + "pairwise_causal_calibration_ece": 0.4530822918880772, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.9185643919214089, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "440577711/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": -0.5741772066455673, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5471109523798312, + "pairwise_causal_calibration_ece": 0.09574619047731159, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07382839572805955, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.203963503241539, + "selected_utility_gain_over_base": -0.005804136395454407, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "tau": -0.09621934768667256 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5207271401629574, + "pairwise_causal_calibration_ece": 0.08641571697989969, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1291832189884023, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "tau": -0.09621934768667256 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5494241983519527, + "pairwise_causal_calibration_ece": 0.2720043730766188, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06231616583540775, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2042645812034607, + "selected_utility_gain_over_base": -0.005503058433532715, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "tau": -0.09621934768667256 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44, + "pairwise_causal_calibration_confidence": 0.5600415333760875, + "pairwise_causal_calibration_ece": 0.12004153337608771, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -1.0643933226145403, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5391718475176117, + "pairwise_causal_calibration_ece": 0.07481690519404037, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.042695474312121, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3181818181818182, + "pairwise_causal_calibration_confidence": 0.5207948539979985, + "pairwise_causal_calibration_ece": 0.20261303581618034, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": -1.0166419733394694, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5640865527534807, + "pairwise_causal_calibration_ece": 0.11448487581794778, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3088799361849384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14155812561511993, + "selected_utility_gain_over_base": 0.07052158564329147, + "selector_regret": 0.022925466299057007, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "tau": -0.09621934768667256 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.541221456647389, + "pairwise_causal_calibration_ece": 0.3269357423616748, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14014647023712193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08595272153615952, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "tau": -0.09621934768667256 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5622977631095711, + "pairwise_causal_calibration_ece": 0.13372633453814248, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5298559867159688, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13704223930835724, + "selected_utility_gain_over_base": 0.06600569933652878, + "selector_regret": 0.15684272348880768, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "tau": -0.09621934768667256 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5408065606528176, + "pairwise_causal_calibration_ece": 0.18366370350996067, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21949047792385512, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.529986560344696, + "selected_utility_gain_over_base": 0.004626452922821045, + "selector_regret": 0.08247500658035278, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "tau": -0.09621934768667256 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5321689975615193, + "pairwise_causal_calibration_ece": 0.3893118547043764, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.08822119814399709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5021255612373352, + "selected_utility_gain_over_base": -0.023234546184539795, + "selector_regret": 0.10797518491744995, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "tau": -0.09621934768667256 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5421874092011935, + "pairwise_causal_calibration_ece": 0.32790169491547927, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.23922942899090938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.38905441761016846, + "selected_utility_gain_over_base": -0.13630568981170654, + "selector_regret": 0.18145644664764404, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "tau": -0.09621934768667256 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5478515031677721, + "pairwise_causal_calibration_ece": 0.30763042969538684, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3376491381273658, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05774742551147938, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03713102079927921, + "selected_utility_gain_over_base": -0.00460987351834774, + "selector_regret": 0.020616404712200165, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1078349556773901, + "tau": -0.09621934768667256 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6964285714285714, + "pairwise_causal_calibration_confidence": 0.5601554432816065, + "pairwise_causal_calibration_ece": 0.13627312814696482, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5271334396681917, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0619367565959692, + "selected_candidate_index": 1.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05393064022064209, + "selected_utility_gain_over_base": 0.012189745903015137, + "selector_regret": 0.00800611637532711, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10364562459290028, + "tau": -0.09621934768667256 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3035714285714286, + "pairwise_causal_calibration_confidence": 0.5877935841757066, + "pairwise_causal_calibration_ece": 0.28542884843534555, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.665136663814353, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08006523177027702, + "selected_candidate_index": 2.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06953043676912785, + "selected_utility_gain_over_base": 0.027789542451500893, + "selector_regret": 0.010534795001149178, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08551714941859245, + "tau": -0.09621934768667256 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5496721466025628, + "pairwise_causal_calibration_ece": 0.15681500374541976, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6023100427879817, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04850691556930542, + "selected_utility_gain_over_base": -0.07106667757034302, + "selector_regret": 0.05927673727273941, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "tau": -0.09621934768667256 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5456989902942814, + "pairwise_causal_calibration_ece": 0.11712756172285277, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6318863252856292, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07379839569330215, + "selected_utility_gain_over_base": -0.04577519744634628, + "selector_regret": 0.038734301924705505, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "tau": -0.09621934768667256 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5764597095564609, + "pairwise_causal_calibration_ece": 0.1499974907415182, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7477968236574296, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08920355886220932, + "selected_utility_gain_over_base": -0.030370034277439117, + "selector_regret": 0.03550797700881958, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "tau": -0.09621934768667256 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5186559158413719, + "pairwise_causal_calibration_ece": 0.33848694130148516, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.028816509714223226, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14302301406860352, + "selected_utility_gain_over_base": -0.0010523796081542969, + "selector_regret": 0.014375895261764526, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "tau": -0.09621934768667256 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5209751256013208, + "pairwise_causal_calibration_ece": 0.1995465541727493, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03719097122411977, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12636350095272064, + "selected_utility_gain_over_base": -0.01771189272403717, + "selector_regret": 0.024023786187171936, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "tau": -0.09621934768667256 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5269018603789154, + "pairwise_causal_calibration_ece": 0.3126161460932011, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09697672498833285, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1574591100215912, + "selected_utility_gain_over_base": 0.013383716344833374, + "selector_regret": 0.014966011047363281, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "tau": -0.09621934768667256 + }, + "672328262/train0": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5369764428991062, + "pairwise_causal_calibration_ece": 0.17730927138660812, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.004905813304151241, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.045009810477495193, + "selected_utility_gain_over_base": -0.060924429446458817, + "selector_regret": 0.014783523976802826, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "tau": -0.09621934768667256 + }, + "672328262/train1": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5336834639813024, + "pairwise_causal_calibration_ece": 0.2479691782670167, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15584019094451163, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "tau": -0.09621934768667256 + }, + "672328262/train2": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5319707668349153, + "pairwise_causal_calibration_ece": 0.17482790969205814, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11650077283620158, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0718381404876709, + "selected_utility_gain_over_base": -0.03409609943628311, + "selector_regret": 0.00032808631658554077, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "tau": -0.09621934768667256 + }, + "701667486/train0": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5330187864606177, + "pairwise_causal_calibration_ece": 0.17587592931776058, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.01726372954718114, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6136738657951355, + "selected_utility_gain_over_base": -0.1776002049446106, + "selector_regret": 0.17424744367599487, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "tau": -0.09621934768667256 + }, + "701667486/train1": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5271169408929481, + "pairwise_causal_calibration_ece": 0.027116940892948138, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.14502256377520684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "tau": -0.09621934768667256 + }, + "701667486/train2": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5417148439395401, + "pairwise_causal_calibration_ece": 0.0699820652511001, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.17712518900425797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.09621934768667256 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5257182607095816, + "pairwise_causal_calibration_ece": 0.00999602500470409, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.952906533401014, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "tau": -0.09621934768667256 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5242087065846202, + "pairwise_causal_calibration_ece": 0.15436272198680845, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.8918433784077185, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "tau": -0.09621934768667256 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5102074375738617, + "pairwise_causal_calibration_ece": 0.11735029471671882, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.738412396559485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "tau": -0.09621934768667256 + }, + "750448296/train0": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.6071428571428572, + "pairwise_causal_calibration_confidence": 0.5484958674304367, + "pairwise_causal_calibration_ece": 0.14575082602783154, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.15402266437593162, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20943730572859445, + "selected_candidate_index": 2.3333333333333335, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7211182241638502, + "selected_utility_gain_over_base": -0.039385102689266205, + "selector_regret": 0.07583838452895482, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1985591898361843, + "tau": -0.09621934768667256 + }, + "750448296/train1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.559706435087612, + "pairwise_causal_calibration_ece": 0.23807978899777407, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0005268387017873799, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22532296180725098, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.3333333333333333, + "selected_utility": 0.1491819222768148, + "selected_utility_gain_over_base": -0.6113214045763016, + "selector_regret": 0.0761410395304362, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1875135501225789, + "tau": -0.09621934768667256 + }, + "750448296/train2": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.5238095238095237, + "pairwise_causal_calibration_confidence": 0.5774293205036009, + "pairwise_causal_calibration_ece": 0.14859956263801863, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.02075587526448386, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2481476068496704, + "selected_candidate_index": 2.6666666666666665, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8722578237454096, + "selected_utility_gain_over_base": 0.11175449689229329, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1598488887151082, + "tau": -0.09621934768667256 + }, + "767747424/train0": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.6964285714285714, + "pairwise_causal_calibration_confidence": 0.5475025060335299, + "pairwise_causal_calibration_ece": 0.16004483653583484, + "pairwise_causal_calibration_pairs": 22.75, + "predicted_margin": 0.23363424930279666, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5789013803005219, + "selected_candidate_index": 4.0, + "selected_success": 0.25, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.5703066997230053, + "selected_utility_gain_over_base": -0.5044020563364029, + "selector_regret": 0.008594680577516556, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10475751012563705, + "tau": -0.09621934768667256 + }, + "767747424/train1": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.4910714285714285, + "pairwise_causal_calibration_confidence": 0.548447365324666, + "pairwise_causal_calibration_ece": 0.13072288927515163, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.17040361845802016, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5825987188145518, + "selected_candidate_index": 2.25, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.056986487004906, + "selected_utility_gain_over_base": -0.01772226905450225, + "selector_regret": 0.022256564814597368, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1008116127923131, + "tau": -0.09621934768667256 + }, + "767747424/train2": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.75, + "execute_generated": 0.75, + "fallback_rate": 0.25, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.48214285714285715, + "pairwise_causal_calibration_confidence": 0.5398024896497756, + "pairwise_causal_calibration_ece": 0.08963847303782024, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.06512356295726199, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5809952197596431, + "selected_candidate_index": 4.5, + "selected_success": 0.25, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.5751176299527287, + "selected_utility_gain_over_base": -0.49959112610667944, + "selector_regret": 0.0058775898069143295, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09709237236529589, + "tau": -0.09621934768667256 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..decc09b746fe1c0fe6164f1fa0acea7aacbaffb7 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.0, + "base_utility": 0.4796089559793472, + "coverage": 0.9333333333333333, + "execute_generated": 0.9333333333333333, + "fallback_rate": 0.06666666666666667, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.44400352733686066, + "pairwise_causal_calibration_confidence": 0.5297947934025979, + "pairwise_causal_calibration_ece": 0.17610616365730683, + "pairwise_causal_calibration_pairs": 27.933333333333334, + "predicted_margin": 0.044647164658061525, + "proposal_oracle_success": 0.13333333333333333, + "proposal_oracle_utility": 0.6601470321416855, + "selected_candidate_index": 4.8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.46326595544815063, + "selected_utility_gain_over_base": -0.016343000531196594, + "selector_regret": 0.19712601602077484, + "success_selector_gap": 0.13333333333333333, + "success_support_gap": 0.8666666666666667, + "support_gap": 1.3398529678583144, + "tau": -0.09621934768667259 + }, + "PickCube-v1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6904761904761904, + "pairwise_causal_calibration_confidence": 0.5548326273106553, + "pairwise_causal_calibration_ece": 0.13565268355755775, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.5743526430578836, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 1.021850009759267, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 0.9781499902407328, + "tau": -0.09621934768667256 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.8032506120204925, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.32, + "pairwise_causal_calibration_accuracy": 0.5367965367965366, + "pairwise_causal_calibration_confidence": 0.5519987946284919, + "pairwise_causal_calibration_ece": 0.14975573664802355, + "pairwise_causal_calibration_pairs": 24.16, + "predicted_margin": 0.051817275904245494, + "proposal_oracle_success": 0.17333333333333334, + "proposal_oracle_utility": 0.46170220340291657, + "selected_candidate_index": 3.3866666666666667, + "selected_success": 0.30666666666666664, + "selected_success_gain_over_base": -0.05333333333333334, + "selected_utility": 0.6904245187963048, + "selected_utility_gain_over_base": -0.1128260932241877, + "selector_regret": 0.02100611601024866, + "success_selector_gap": 0.0, + "success_support_gap": 0.3466666666666667, + "support_gap": 0.6881442584097386, + "tau": -0.09621934768667265 + }, + "PushCube-v1": { + "base_success": 0.8333333333333334, + "base_utility": 1.6734557673335075, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.16666666666666666, + "pairwise_causal_calibration_accuracy": 0.4658470723176606, + "pairwise_causal_calibration_confidence": 0.5396225164665516, + "pairwise_causal_calibration_ece": 0.14011534783179128, + "pairwise_causal_calibration_pairs": 23.833333333333332, + "predicted_margin": -0.8998105005052524, + "proposal_oracle_success": 0.5555555555555556, + "proposal_oracle_utility": 1.1790093589160178, + "selected_candidate_index": 3.6666666666666665, + "selected_success": 0.8333333333333334, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.6734557673335075, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32654423266649246, + "success_selector_gap": 0.16666666666666666, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6220440144340197, + "tau": -0.09621934768667259 + }, + "StackCube-v1": { + "base_success": 0.09090909090909091, + "base_utility": 0.5003613992170854, + "coverage": 0.7272727272727273, + "execute_generated": 0.7272727272727273, + "fallback_rate": 0.2727272727272727, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6060606060606061, + "pairwise_causal_calibration_accuracy": 0.5692640692640694, + "pairwise_causal_calibration_confidence": 0.5286780647437916, + "pairwise_causal_calibration_ece": 0.14904052207511279, + "pairwise_causal_calibration_pairs": 27.818181818181817, + "predicted_margin": 0.04387425666638764, + "proposal_oracle_success": 0.18181818181818182, + "proposal_oracle_utility": 0.634040683056369, + "selected_candidate_index": 3.9696969696969697, + "selected_success": 0.06060606060606061, + "selected_success_gain_over_base": -0.030303030303030304, + "selected_utility": 0.4272875478773406, + "selected_utility_gain_over_base": -0.0730738513397448, + "selector_regret": 0.24322165729421558, + "success_selector_gap": 0.15151515151515152, + "success_support_gap": 0.8181818181818182, + "support_gap": 1.365959316943631, + "tau": -0.09621934768667252 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..3f6dc4c5dedd947db16b1d01b23d5a9e498f7aa2 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.6111 & 0.3889 & 0.3333 & 0.2986 & 0.2222 & 0.3611 & 0.5139 & 0.0694 & 0.0145 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e79cea792beb2cff5662834793a0057739905041 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_perdim_trainmax_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=10.0 +tau=-0.096219 +calibration_input=runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..efc423b79a05746d2fdca54ae6adcf67167dd2c1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test --feature-set context_tangent --target success --k 8 --bootstrap-samples 1000 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d66fb821fa4d4d33f868ba467254a3587df21168 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: context_tangent +fit_objective: pointwise +k: 8 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: None +success_bonus: 1.0 +target: success +threshold_scope: global diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d91fbacd145085f2d9a143f07cf1791ff492f9fc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "130d86b8c5ba5dc901ee48835454693bf42c361803c01f6b2d80590cb2493491", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "88d4bfe8c95f8d1579cbe59096cf665dc5f2dfdd99481a386a25e1f3cfa2b09d", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..58e60527653c049a1edc75cf9002db1286feb6ae --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9f1999238aef5a156cfccb03d5abdf7c6a1f1565 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics.json @@ -0,0 +1,9155 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.0625, + "fallback_rate": 0.9375, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3125, + "selected_success_gain_over_base": 0.013888888888888888, + "selected_utility": 0.7703299819129623, + "selector_regret": 0.058519456173396774, + "success_selector_gap": 0.03125, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_summary": { + "base_success": 0.2986111111111111, + "base_utility": 0.748683326644823, + "coverage": 0.0625, + "fallback_rate": 0.9375, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.2638888888888889, + "pairwise_causal_calibration_accuracy": 0.5348460291734197, + "pairwise_causal_calibration_confidence": 0.516140452233134, + "pairwise_causal_calibration_ece": 0.01870557694028568, + "pairwise_causal_calibration_pairs": 7404.0, + "proposal_oracle_success": 0.1909722222222222, + "proposal_oracle_utility": 0.546237602631057, + "selected_success": 0.3125, + "selected_success_gain_over_base": 0.013888888888888888, + "selected_utility": 0.7703299819129623, + "selector_regret": 0.058519456173396774, + "success_selector_gap": 0.03125, + "success_support_gap": 0.4826388888888889, + "support_gap": 0.88937310987482 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_perdim_trainmax_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.3333333333333333, + "base_utility": 0.8338337515791258, + "coverage": 0.09027777777777778, + "fallback_rate": 0.9097222222222222, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.3611111111111111, + "pairwise_causal_calibration_accuracy": 0.5024576734025122, + "pairwise_causal_calibration_confidence": 0.5176934816531578, + "pairwise_causal_calibration_ece": 0.015235808250645553, + "pairwise_causal_calibration_pairs": 3662.0, + "proposal_oracle_success": 0.2222222222222222, + "proposal_oracle_utility": 0.6232009150553495, + "selected_success": 0.2777777777777778, + "selected_success_gain_over_base": -0.05555555555555555, + "selected_utility": 0.731696648730172, + "selector_regret": 0.16677976430704197, + "success_selector_gap": 0.09027777777777778, + "success_support_gap": 0.5138888888888888, + "support_gap": 0.9091424554741631 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.182546837171281, + 1.8836221616495297, + -1.0601087908747155e-18, + 3.5, + 0.125, + 1.2092013888888888, + 0.7583173435164257, + 1.9059527259604592, + 8.0, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.005174291938999986, + -0.023257080610021998, + -0.44133986928105434, + 0.7372549019607866, + 0.4918300653594769, + 0.6157952069716777, + 0.36546840958604854, + -0.06666666666666653, + 0.36258169934639883, + -0.4588235294117674, + 0.1346405228758154, + 0.3127450980392166, + -0.5231481481481283, + -0.5494008714596817, + 0.5098039215686088, + -0.08622004357298363, + 1.0, + 0.2707248263888889, + 0.2306857638888889, + 0.15460957213589507, + -0.002221525274055441, + 0.4404452632491787, + -0.0732957456914139, + 0.1185007049029486, + 0.28929648765349864, + 0.37639624912602204, + 0.289247130454492, + 0.08760700280981837, + 0.021523393619443394, + 0.0013782045918762581, + 0.07315826858070472, + -0.36818566475348136, + 0.08511420341286187, + 0.10109417613744477, + 0.03422285977952318, + 0.07723466354137701, + 0.04658102504779688, + 0.17487594802838025, + -0.1345592589544443, + 0.07206917453893563, + 0.4632009652947697, + 0.13900196984549337, + 0.6286754180263314, + 0.4662843597309095, + 0.7146745238246189, + 1.1377109955808717, + 0.7123874865420576, + 0.42250050161641817, + 0.18250456237243876, + 0.34622435484521297, + 0.4512131481436275, + 0.6555436916565264, + 0.8817456337920804, + 0.6499884038219332, + 0.35018353749344694, + 0.1856164064041271, + 0.4309610689589236, + 0.3701181374425586, + 0.5064969249263717, + 0.9176829091027483, + 0.5763792715798546 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "target_task_hash_00", + "target_task_hash_01", + "target_task_hash_02", + "target_task_hash_03", + "target_task_hash_04", + "target_task_hash_05", + "target_task_hash_06", + "target_task_hash_07", + "source_task_hash_00", + "source_task_hash_01", + "source_task_hash_02", + "source_task_hash_03", + "source_task_hash_04", + "source_task_hash_05", + "source_task_hash_06", + "source_task_hash_07", + "instruction_hash_00", + "instruction_hash_01", + "instruction_hash_02", + "instruction_hash_03", + "instruction_hash_04", + "instruction_hash_05", + "instruction_hash_06", + "instruction_hash_07", + "source_target_same_task", + "instruction_chars_per_128", + "instruction_words_per_32", + "tangent_00", + "tangent_01", + "tangent_02", + "tangent_03", + "tangent_04", + "tangent_05", + "tangent_06", + "tangent_07", + "tangent_08", + "tangent_09", + "tangent_10", + "tangent_11", + "tangent_12", + "tangent_13", + "tangent_14", + "tangent_15", + "tangent_16", + "tangent_17", + "tangent_18", + "tangent_19", + "tangent_20", + "abs_tangent_00", + "abs_tangent_01", + "abs_tangent_02", + "abs_tangent_03", + "abs_tangent_04", + "abs_tangent_05", + "abs_tangent_06", + "abs_tangent_07", + "abs_tangent_08", + "abs_tangent_09", + "abs_tangent_10", + "abs_tangent_11", + "abs_tangent_12", + "abs_tangent_13", + "abs_tangent_14", + "abs_tangent_15", + "abs_tangent_16", + "abs_tangent_17", + "abs_tangent_18", + "abs_tangent_19", + "abs_tangent_20" + ], + "feature_set": "context_tangent", + "feature_std": [ + 1.0, + 2.4903288136152946, + 2.1478743840800227, + 0.999998700632758, + 2.29128884747792, + 0.3307199138830738, + 1.490882562848811, + 0.3689689239790769, + 0.7063143432867478, + 1e-06, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37569767092837797, + 0.6050101452051985, + 0.27519889073248605, + 0.24940029539540393, + 0.12420765409520347, + 0.27524742237325, + 0.3631166553339116, + 0.2513657655812379, + 0.37487873363847424, + 0.3443939715604908, + 0.32513795823134695, + 0.3102098787644046, + 0.4154730332740044, + 0.3938110555196154, + 0.4628208676716171, + 0.7177468630079228, + 1e-06, + 0.03103759015348271, + 0.02828865844409628, + 0.7164333963112022, + 0.19212642197808302, + 0.697210408452911, + 0.7046809935624476, + 0.9225902325860902, + 1.4625751985388367, + 1.0522456240259936, + 0.6818443950002842, + 0.2884065822442686, + 0.5388119913722541, + 0.7247091653772777, + 0.8579675721375568, + 1.1446825413579313, + 1.0496567177779326, + 0.5457951029759756, + 0.2892210779882745, + 0.6832853149096569, + 0.5792173381920694, + 0.7114807110238744, + 1.1989477833750786, + 0.9428868665587309, + 0.5680013204490734, + 0.13264927674671176, + 0.533723969267122, + 0.5334099008762095, + 0.595361087797577, + 0.9635514317546396, + 0.8610451069806081, + 0.608332853444767, + 0.2398860798282785, + 0.4134127040051782, + 0.5671081498003836, + 0.5583210153106908, + 0.8175468104254054, + 0.8285760589919584, + 0.43067805460220965, + 0.2244247227136431, + 0.5358323216821982, + 0.4479674302815383, + 0.5293835095689982, + 0.7832236132310876, + 0.7496773692207523 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 2304, + "num_fit_rows": 2304, + "num_pairwise_rows": 0, + "num_pointwise_rows": 2304, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 8, + "num_calibration_candidates": 2304, + "num_calibration_rows": 288, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5348460291734197, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.018241228640186846, + "accuracy": 0.5342261904761905, + "confidence": 0.5159849618360036, + "count": 7392, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.30474412980116905, + "accuracy": 0.9166666666666666, + "confidence": 0.6119225368654976, + "count": 12, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.01870557694028568, + "mean_confidence": 0.516140452233134, + "n_bins": 10, + "num_pairs": 7404, + "num_rows": 288 + }, + "eval": { + "accuracy": 0.5024576734025122, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.014197486971308404, + "accuracy": 0.5030170049369171, + "confidence": 0.5172144919082255, + "count": 3646, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.25184326977959826, + "accuracy": 0.375, + "confidence": 0.6268432697795983, + "count": 16, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.015235808250645553, + "mean_confidence": 0.5176934816531578, + "n_bins": 10, + "num_pairs": 3662, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5090347138321765, + "pairwise_causal_calibration_ece": 0.044748999546462165, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18515916980019373, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "seed": "2012211287/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5119848106472845, + "pairwise_causal_calibration_ece": 0.19055623921871306, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.005683666382895997, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "seed": "307068879/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5092992512408128, + "pairwise_causal_calibration_ece": 0.15215639409795562, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20584099091859903, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5141263992957481, + "pairwise_causal_calibration_ece": 0.12126925643860526, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2141408309665893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "seed": "608016290/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08710145950317383, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.510976779082865, + "pairwise_causal_calibration_ece": 0.024737506631420714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.006895278681057632, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "seed": "441046872/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5308364411582983, + "pairwise_causal_calibration_ece": 0.38797929830115546, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4948526502859529, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "seed": "1522272686/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "task_id": "PickCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.6873128160326993, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5257193927291514, + "pairwise_causal_calibration_ece": 0.3828622498720085, + "pairwise_causal_calibration_pairs": 7.0, + "predicted_margin": 0.31288945257356704, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5120775991497745, + "pairwise_causal_calibration_ece": 0.05935097227879693, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21845740942913472, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "seed": "1461069713/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5245031538200983, + "pairwise_causal_calibration_ece": 0.22549684617990173, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3689242565019341, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.044337254017591476, + "seed": "626486813/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01164356991648674, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2090834565460682, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5096744733127383, + "pairwise_causal_calibration_ece": 0.0617540981158331, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5197254225286851, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14525407552719116, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8547459244728088, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5165144952070263, + "pairwise_causal_calibration_ece": 0.2334855047929737, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.02720919346098577, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "seed": "672328262/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5304661285476823, + "pairwise_causal_calibration_ece": 0.18381958573803203, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6991682352849198, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2748139500617981, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.24293343722820282, + "selected_utility_gain_over_base": 0.04594218730926514, + "selector_regret": 0.031880512833595276, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5223926438814469, + "pairwise_causal_calibration_ece": 0.022392643881446905, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2281877708298647, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14440619945526123, + "seed": "278008524/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0834110826253891, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5068891153127965, + "pairwise_causal_calibration_ece": 0.17168231325863215, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05982643176183276, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "seed": "1845489013/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5167491796572531, + "pairwise_causal_calibration_ece": 0.3024634653715388, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0889017598958553, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07115759700536728, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05078810453414917, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02036949247121811, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.006586454808712006, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5120280305937869, + "pairwise_causal_calibration_ece": 0.05940054083478452, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.24885553336134392, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.011163711547851562, + "seed": "767747424/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2515503764152527, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5316537240199258, + "pairwise_causal_calibration_ece": 0.14691770455150277, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15837838927275857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "seed": "585904130/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09344705194234848, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5150349488116895, + "pairwise_causal_calibration_ece": 0.1578920916688324, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18927476177277072, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "seed": "141984615/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.531161762965516, + "pairwise_causal_calibration_ece": 0.13830462010837313, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.29167751936252423, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25842753052711487, + "seed": "750448296/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.19032493233680725, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7415724694728851, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5172753273454199, + "pairwise_causal_calibration_ece": 0.12441818448827707, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19058677054848244, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "seed": "636680137/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5288589588010555, + "pairwise_causal_calibration_ece": 0.13600181594391264, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1260024655502223, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.029627859592437744, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.16747966408729553, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6923076923076923, + "pairwise_causal_calibration_confidence": 0.5390208488024293, + "pairwise_causal_calibration_ece": 0.153286843505263, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.23249138223608884, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5291333104927968, + "pairwise_causal_calibration_ece": 0.13627616763565392, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07343156967546631, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "seed": "2038594191/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0574311688542366, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5107433876562132, + "pairwise_causal_calibration_ece": 0.16782804091521542, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2044536631964687, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13244229555130005, + "seed": "750448296/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09154719114303589, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.24444645453630715, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5120181933719066, + "pairwise_causal_calibration_ece": 0.11916105051476372, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1795266808160827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "seed": "747260437/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.20225667035712927, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5465855152184887, + "pairwise_causal_calibration_ece": 0.20341448478151117, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4099932021305159, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04541558027267456, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5138796533820333, + "pairwise_causal_calibration_ece": 0.057548918046538056, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0577847490536553, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "seed": "1822210471/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5280778360058722, + "pairwise_causal_calibration_ece": 0.007636449708413484, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5565962131677729, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23744209110736847, + "seed": "750448296/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7625579088926315, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5163938105237615, + "pairwise_causal_calibration_ece": 0.12646333233338136, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5586790894555644, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "seed": "440577711/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5213088389133544, + "pairwise_causal_calibration_ece": 0.1284516960562116, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.8928362396780647, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "seed": "1324642411/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14966940879821777, + "selected_utility_gain_over_base": -1.8503305912017822, + "selector_regret": 0.060336410999298096, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5223174936773409, + "pairwise_causal_calibration_ece": 0.15625393489408768, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25334075772729825, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.507207152590294, + "pairwise_causal_calibration_ece": 0.00720715259029403, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.006272514778440541, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "seed": "1261930456/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.28, + "pairwise_causal_calibration_confidence": 0.5167159497943329, + "pairwise_causal_calibration_ece": 0.23671594979433286, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.40670225613190863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5131988287429138, + "pairwise_causal_calibration_ece": 0.15605597160005663, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7158970211825535, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "seed": "1480856810/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12667706608772278, + "selected_utility_gain_over_base": -1.8733229339122772, + "selector_regret": 0.015103071928024292, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5035728957399334, + "pairwise_causal_calibration_ece": 0.13928424711720955, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08547837727895316, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "seed": "2061635888/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5125805300949516, + "pairwise_causal_calibration_ece": 0.16599089847647697, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03266312735678528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "seed": "655884605/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01332351565361023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5120817381301231, + "pairwise_causal_calibration_ece": 0.04779602384440884, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11841214667429321, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "seed": "1113371261/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5120526217939588, + "pairwise_causal_calibration_ece": 0.05937594963461257, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12075832280916365, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "seed": "1578990937/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.5069861149914309, + "pairwise_causal_calibration_ece": 0.13301388500856914, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.1645139076256539, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5129474725633024, + "pairwise_causal_calibration_ece": 0.09419538457955468, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22532734972882426, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "seed": "211245842/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3979237377643585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5084242760212154, + "pairwise_causal_calibration_ece": 0.20586143826449887, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14682773551354072, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "seed": "1151410346/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3400209844112396, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5156896062548352, + "pairwise_causal_calibration_ece": 0.05573896517373622, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07612377904669673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "seed": "1324133122/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5106631871119919, + "pairwise_causal_calibration_ece": 0.06076538431657952, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18005163348657682, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "seed": "701667486/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5149085793100113, + "pairwise_causal_calibration_ece": 0.08633715073858278, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22150303258309284, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5125355587394795, + "pairwise_causal_calibration_ece": 0.20175015554623477, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19337117648183325, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "seed": "1999022574/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.5141496618096716, + "pairwise_causal_calibration_ece": 0.2919274395874494, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.13504095940734823, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2717533111572266, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5075913402530835, + "pairwise_causal_calibration_ece": 0.0433056259673692, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.01192475167673359, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "seed": "307068879/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.506518118161019, + "pairwise_causal_calibration_ece": 0.11366097530387614, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19554991093790747, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "seed": "1073161778/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5114864515125813, + "pairwise_causal_calibration_ece": 0.36862930865543847, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19953791912822288, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "seed": "608016290/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08474063873291016, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5088684124078007, + "pairwise_causal_calibration_ece": 0.06256015902077072, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03548490283103705, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "seed": "441046872/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5263848235942014, + "pairwise_causal_calibration_ece": 0.24067053787991566, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3317013229350659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "seed": "1522272686/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "task_id": "PickCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.9835802031786316, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.517416825601227, + "pairwise_causal_calibration_ece": 0.05401174582734436, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.021681528943211334, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5159485485371296, + "pairwise_causal_calibration_ece": 0.05548002289144183, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30554439462118227, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "seed": "1461069713/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5207140020481635, + "pairwise_causal_calibration_ece": 0.020714002048163538, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3761401160884725, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06662949174642563, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03393580764532089, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18679121881723404, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5220705265713083, + "pairwise_causal_calibration_ece": 0.1292133837141654, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7459948550290747, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1511060744524002, + "seed": "2054899126/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12261688709259033, + "selected_utility_gain_over_base": -1.8773831129074097, + "selector_regret": 0.028489187359809875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8488939255475998, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07142857142857142, + "pairwise_causal_calibration_confidence": 0.5183234734272898, + "pairwise_causal_calibration_ece": 0.4468949019987184, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.047096220474437275, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "seed": "672328262/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5199403513200307, + "pairwise_causal_calibration_ece": 0.16279749417717354, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5392752394089562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2738197147846222, + "seed": "767747424/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07682846486568451, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5243015603781166, + "pairwise_causal_calibration_ece": 0.2256984396218834, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17135611709288995, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24920666217803955, + "seed": "278008524/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5120112216374724, + "pairwise_causal_calibration_ece": 0.08343979306604382, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0787154239384655, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "seed": "1845489013/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5138165271659146, + "pairwise_causal_calibration_ece": 0.12904061569122827, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20779619260692878, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05724402144551277, + "seed": "626486813/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05078810453414917, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006455916911363602, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.020500030368566513, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5157342854045418, + "pairwise_causal_calibration_ece": 0.019980000309743895, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09492165683011476, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.013422667980194092, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24929141998291016, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5109791799445677, + "pairwise_causal_calibration_ece": 0.1538363228017105, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07464422139595693, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "seed": "585904130/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08595272153615952, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5209787160422698, + "pairwise_causal_calibration_ece": 0.05669300175655545, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18318239803192132, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5430286758741075, + "pairwise_causal_calibration_ece": 0.20697132412589253, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25312753645605457, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2347063571214676, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.16660375893115997, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7652936428785324, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.521068070904193, + "pairwise_causal_calibration_ece": 0.1282109280470502, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12268453237476608, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "seed": "636680137/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.57060357404702, + "pairwise_causal_calibration_ece": 0.24917500261844866, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4218554685316619, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04315249249339104, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15395503118634224, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5136001990069363, + "pairwise_causal_calibration_ece": 0.3078283724216351, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.037632419519960006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "seed": "1375492786/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5272185314816756, + "pairwise_causal_calibration_ece": 0.11563861137546727, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11157849082379764, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09524012356996536, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5308481795331372, + "pairwise_causal_calibration_ece": 0.06656246524742293, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.36796809714218315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23850953578948975, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.025102153420448303, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.4196819897905272, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5088316663078858, + "pairwise_causal_calibration_ece": 0.02688261940639991, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17198143779324082, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.22597190961061323, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5489097792855695, + "pairwise_causal_calibration_ece": 0.3082330778572876, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4366493356305453, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1363367885351181, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5083982289146939, + "pairwise_causal_calibration_ece": 0.07982680034326534, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07212192716463756, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5178200699806214, + "pairwise_causal_calibration_ece": 0.017894215733664343, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.48002621299586945, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2027529925107956, + "seed": "750448296/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7972470074892044, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5239216972771956, + "pairwise_causal_calibration_ece": 0.5239216972771956, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.660329471290032, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.21395862102508545, + "selected_utility_gain_over_base": -1.7860413789749146, + "selector_regret": 1.7860413789749146, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.18518518518518517, + "pairwise_causal_calibration_confidence": 0.5212524000863837, + "pairwise_causal_calibration_ece": 0.33606721490119856, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.7531487720723857, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.17775490880012512, + "selected_utility_gain_over_base": -1.8222450911998749, + "selector_regret": 1.8222450911998749, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5184372952384458, + "pairwise_causal_calibration_ece": 0.23156270476155416, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.27780618403492885, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "seed": "1790188787/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.509870803335365, + "pairwise_causal_calibration_ece": 0.025843482378920646, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.077102113380139, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "seed": "1261930456/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.09090909090909091, + "pairwise_causal_calibration_confidence": 0.5152174445598239, + "pairwise_causal_calibration_ece": 0.42430835365073294, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.46272796115382003, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5316435268709924, + "pairwise_causal_calibration_ece": 0.13878638401384952, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7444206849830362, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.184015691280365, + "selected_utility_gain_over_base": -1.815984308719635, + "selector_regret": 1.815984308719635, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5095019143713128, + "pairwise_causal_calibration_ece": 0.27621237134297294, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03930678750007065, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "seed": "2061635888/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.004257768392562866, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.506450790055417, + "pairwise_causal_calibration_ece": 0.10069206708744005, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03338313576240709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "seed": "655884605/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006311893463134766, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5114294633513058, + "pairwise_causal_calibration_ece": 0.1671419652201228, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.111487318938856, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "seed": "1113371261/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.517049827852834, + "pairwise_causal_calibration_ece": 0.37580731500430886, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15613350902346218, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "seed": "1578990937/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0033643990755081177, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5077947588157862, + "pairwise_causal_calibration_ece": 0.10779475881578615, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.12250900335942658, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5130796019472789, + "pairwise_causal_calibration_ece": 0.12977754090986404, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22424307526810303, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "seed": "211245842/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.017371907830238342, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5116087729702029, + "pairwise_causal_calibration_ece": 0.047323058684488606, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1505309920780672, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "seed": "1151410346/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.015453755855560303, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5077543727318965, + "pairwise_causal_calibration_ece": 0.2422456272681035, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21872963666248138, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "seed": "1324133122/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.00014631450176239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5098775552568305, + "pairwise_causal_calibration_ece": 0.02583673045745516, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12475135370966416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "seed": "701667486/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5120484995354441, + "pairwise_causal_calibration_ece": 0.04776278524972977, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12051079262473083, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5081622486924189, + "pairwise_causal_calibration_ece": 0.07959082012099034, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09814524693113953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5108011563830763, + "pairwise_causal_calibration_ece": 0.18937258495450487, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12797764508486972, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "seed": "2012211287/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.50920166710305, + "pairwise_causal_calibration_ece": 0.09794119003980706, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.003886783551851819, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5082470514492481, + "pairwise_causal_calibration_ece": 0.2939613371635338, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19441156679128385, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274482011795044, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5207029573796137, + "pairwise_causal_calibration_ece": 0.41356010023675654, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.24661642979812717, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "seed": "608016290/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0451507568359375, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5127281442153561, + "pairwise_causal_calibration_ece": 0.16584328435607254, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0013491370718414428, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "seed": "441046872/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "task_id": "LiftPegUpright-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5271177549323166, + "pairwise_causal_calibration_ece": 0.1157393879248263, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3719790973396491, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1522272686/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PickCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.9536859431190323, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5292583746692512, + "pairwise_causal_calibration_ece": 0.029258374669251186, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20185121316938343, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5323058563946664, + "pairwise_causal_calibration_ece": 0.07483700074819066, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5338556706581634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "seed": "1461069713/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.032693684101104736, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5396018963707968, + "pairwise_causal_calibration_ece": 0.21817332494222536, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.37112685427010883, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10755517333745956, + "seed": "626486813/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.032693684101104736, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07486148923635483, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1458655372262001, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5259225813730811, + "pairwise_causal_calibration_ece": 0.22407741862691888, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.6712548039490285, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13674892485141754, + "seed": "2054899126/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12255460023880005, + "selected_utility_gain_over_base": -1.8774453997612, + "selector_regret": 0.014194324612617493, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8632510751485825, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5090400478771011, + "pairwise_causal_calibration_ece": 0.02667423783718459, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0702178669924084, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "seed": "672328262/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19699124991893768, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5092895680546312, + "pairwise_causal_calibration_ece": 0.06213900337394018, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.45484847993038013, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.25252875685691833, + "seed": "767747424/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19699124991893768, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05553750693798065, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.25001442432403564, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.514035350060809, + "pairwise_causal_calibration_ece": 0.23596464993919097, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.054928816762619126, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.31406447291374207, + "seed": "278008524/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.25001442432403564, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06405004858970642, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5049839841564138, + "pairwise_causal_calibration_ece": 0.29069826987069947, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.003957935292008923, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "seed": "1845489013/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05078810453414917, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5269536006440666, + "pairwise_causal_calibration_ece": 0.348382172072638, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21039713585500167, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05257529020309448, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05078810453414917, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0017871856689453125, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.025168761610984802, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07142857142857142, + "pairwise_causal_calibration_confidence": 0.5091431454934968, + "pairwise_causal_calibration_ece": 0.4377145740649254, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17292103844191242, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.01747530698776245, + "seed": "767747424/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2452387809753418, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5257097365720064, + "pairwise_causal_calibration_ece": 0.06142402228629207, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12634263058484524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.22284842282533646, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5266278793488347, + "pairwise_causal_calibration_ece": 0.38377073649169186, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.114212135741804, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "seed": "141984615/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.06810259819030762, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5420182446494998, + "pairwise_causal_calibration_ece": 0.13655318392192883, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4495006342216108, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.4033660888671875, + "seed": "750448296/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06810259819030762, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3352634906768799, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.5966339111328125, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5248912478073117, + "pairwise_causal_calibration_ece": 0.20346267637874021, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3268104640308388, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "seed": "636680137/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005137942731380463, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10184377431869507, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5267665045479556, + "pairwise_causal_calibration_ece": 0.06248079026224129, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.06093237599406439, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053976815193891525, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10184377431869507, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14313070848584175, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5233704587833103, + "pairwise_causal_calibration_ece": 0.15520096978811826, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17970484878502413, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "seed": "1375492786/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5202926621233347, + "pairwise_causal_calibration_ece": 0.05600694783762039, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04359967653884958, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.15376771241426468, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21340738236904144, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5155898129819602, + "pairwise_causal_calibration_ece": 0.020124472732325538, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12215547136088006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17124910652637482, + "seed": "750448296/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21340738236904144, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05274038016796112, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.22933539141955228, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5073520007934037, + "pairwise_causal_calibration_ece": 0.3497908563494534, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1459627586815193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "seed": "747260437/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.4494976891690949, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5156532908328474, + "pairwise_causal_calibration_ece": 0.0914895663100097, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.33492807900387084, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "seed": "1549324805/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04154062271118164, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5174993924353067, + "pairwise_causal_calibration_ece": 0.26749939243530674, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0031507691650567438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "seed": "1822210471/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5228146754897652, + "pairwise_causal_calibration_ece": 0.012899610224520464, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4833920160221713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1698276251554489, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.830172374844551, + "task_id": "PullCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.6996994006285734, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.16666666666666666, + "pairwise_causal_calibration_confidence": 0.527095259436081, + "pairwise_causal_calibration_ece": 0.3604285927694144, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": 0.6491618422591205, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14906781911849976, + "selected_utility_gain_over_base": -1.8509321808815002, + "selector_regret": 1.8509321808815002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.5109241069247938, + "pairwise_causal_calibration_ece": 0.17089407489338793, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.18703346552659444, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5205192822349104, + "pairwise_causal_calibration_ece": 0.12766213937776755, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09402188754699127, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "seed": "1261930456/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.09090909090909091, + "pairwise_causal_calibration_confidence": 0.513267600062783, + "pairwise_causal_calibration_ece": 0.4223585091536921, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.4020750348567389, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5203685487667845, + "pairwise_causal_calibration_ece": 0.08677430837607258, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5127510371288293, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "seed": "1480856810/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "task_id": "PushCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5079491341690449, + "pairwise_causal_calibration_ece": 0.24205086583095514, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.008004389977088786, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "seed": "2061635888/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.505842678549458, + "pairwise_causal_calibration_ece": 0.29155696426374367, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07788535796650367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "seed": "655884605/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.028349727392196655, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5208647930155366, + "pairwise_causal_calibration_ece": 0.05056377841303483, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1637967593188533, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "seed": "1113371261/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5121887042635905, + "pairwise_causal_calibration_ece": 0.09495415287926656, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14872603579685487, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "seed": "1578990937/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018326029181480408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5080683702895407, + "pairwise_causal_calibration_ece": 0.06336020113903074, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1805548023297982, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "seed": "1509980664/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5205888094479262, + "pairwise_causal_calibration_ece": 0.12226833340921672, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2206154553304457, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "seed": "211245842/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.029980570077896118, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5093949827060259, + "pairwise_causal_calibration_ece": 0.0977478744368312, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15717334321041485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "seed": "1151410346/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3345368802547455, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5106739762139129, + "pairwise_causal_calibration_ece": 0.08210254764248431, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0939715257270842, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "seed": "1324133122/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013924404978752136, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5194690186587149, + "pairwise_causal_calibration_ece": 0.08767383848414223, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2642366239473226, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5130901931270576, + "pairwise_causal_calibration_ece": 0.08451876455562907, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15822655084813653, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "seed": "412102563/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5126369132288152, + "pairwise_causal_calibration_ece": 0.20164880105689909, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1565539695903909, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "seed": "1999022574/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "task_id": "StackCube-v1", + "tau": 0.5618318497586814, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 0.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3958333333333333, + "low": 0.2277777777777778, + "mean": 0.3083333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3083333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8053333333333332, + "low": 0.10836363636363637, + "mean": 0.45684848484848484, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.45684848484848484, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.4097222222222222, + "low": 0.2569444444444444, + "mean": 0.3333333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9566364926286041, + "low": 0.6609309806488455, + "mean": 0.8028034761858482, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.8028034761858482, + "per_seed": { + "1073161778/train0": 0.725517988204956, + "1073161778/train1": 0.725517988204956, + "1073161778/train2": 0.725517988204956, + "1113371261/train0": 0.11109928786754608, + "1113371261/train1": 0.11109928786754608, + "1113371261/train2": 0.11109928786754608, + "1151410346/train0": 0.2284735143184662, + "1151410346/train1": 0.2284735143184662, + "1151410346/train2": 0.2284735143184662, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.084227055311203, + "1324133122/train2": 0.084227055311203, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.16783273220062256, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.22852759063243866, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.17249532043933868, + "1549324805/train1": 0.17249532043933868, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21426597237586975, + "1578990937/train1": 0.21426597237586975, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.09442546218633652, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.09442546218633652, + "1845489013/train0": 0.11255800724029541, + "1845489013/train1": 0.11255800724029541, + "1845489013/train2": 0.11255800724029541, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.09504059702157974, + "2038594191/train1": 0.09504059702157974, + "2038594191/train2": 0.09504059702157974, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.08620692789554596, + "2061635888/train1": 0.08620692789554596, + "2061635888/train2": 0.08620692789554596, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.1250072121620178, + "278008524/train1": 1.1250072121620178, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.20915235579013824, + "307068879/train1": 0.20915235579013824, + "307068879/train2": 0.20915235579013824, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.2097676396369934, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2097676396369934, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.07103653997182846, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.07103653997182846, + "608016290/train0": 0.525360107421875, + "608016290/train1": 0.525360107421875, + "608016290/train2": 0.525360107421875, + "626486813/train0": 0.04174089431762695, + "626486813/train1": 0.04174089431762695, + "626486813/train2": 0.04174089431762695, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.1440753936767578, + "655884605/train2": 0.1440753936767578, + "672328262/train0": 0.10593423992395401, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.10593423992395401, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7605033268531164, + "750448296/train1": 0.7605033268531164, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0747087560594082, + "767747424/train1": 1.0747087560594082, + "767747424/train2": 1.0747087560594082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6347634333101186, + "low": 0.5526382644826715, + "mean": 1.0913353469100866, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.0913353469100866, + "per_task": { + "LiftPegUpright-v1": 0.4796089559793472, + "PickCube-v1": 2.0, + "PullCube-v1": 0.8032506120204925, + "PushCube-v1": 1.6734557673335075, + "StackCube-v1": 0.5003613992170854 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9699174793851044, + "low": 0.6952519603073597, + "mean": 0.8338337515791258, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14791666666666667, + "low": 0.04791666666666667, + "mean": 0.09375, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.09375, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.24933333333333332, + "low": 0.0, + "mean": 0.09377777777777778, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.09377777777777778, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.0 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.1388888888888889, + "low": 0.041666666666666664, + "mean": 0.09027777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14791666666666667, + "low": 0.04791666666666667, + "mean": 0.09375, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.09375, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.24933333333333332, + "low": 0.0, + "mean": 0.09377777777777778, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.09377777777777778, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.0 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.1388888888888889, + "low": 0.041666666666666664, + "mean": 0.09027777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9520833333333333, + "low": 0.8520833333333333, + "mean": 0.90625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.90625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 1.0, + "750448296/train1": 1.0, + "750448296/train2": 1.0, + "767747424/train0": 0.75, + "767747424/train1": 1.0, + "767747424/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.7506666666666667, + "mean": 0.9062222222222222, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9062222222222222, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.92, + "PushCube-v1": 0.6111111111111112, + "StackCube-v1": 1.0 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.9583333333333334, + "low": 0.8611111111111112, + "mean": 0.9097222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.45138888888888895, + "low": 0.2833333333333333, + "mean": 0.3631944444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3631944444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 1.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4609696969696969, + "low": 0.1, + "mean": 0.2852121212121212, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2852121212121212, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.6060606060606061 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.4444444444444444, + "low": 0.2847222222222222, + "mean": 0.3611111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5346777320699734, + "low": 0.4641124616124617, + "mean": 0.4997743763261006, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.4997743763261006, + "per_seed": { + "1073161778/train0": 0.35714285714285715, + "1073161778/train1": 0.39285714285714285, + "1073161778/train2": 0.21428571428571427, + "1113371261/train0": 0.4642857142857143, + "1113371261/train1": 0.6785714285714286, + "1113371261/train2": 0.5714285714285714, + "1151410346/train0": 0.7142857142857143, + "1151410346/train1": 0.4642857142857143, + "1151410346/train2": 0.6071428571428571, + "1261930456/train0": 0.5, + "1261930456/train1": 0.5357142857142857, + "1261930456/train2": 0.39285714285714285, + "1324133122/train0": 0.5714285714285714, + "1324133122/train1": 0.75, + "1324133122/train2": 0.42857142857142855, + "1324642411/train0": 0.39285714285714285, + "1324642411/train1": 0.18518518518518517, + "1324642411/train2": 0.16666666666666666, + "1375492786/train0": 0.6923076923076923, + "1375492786/train1": 0.8214285714285714, + "1375492786/train2": 0.6785714285714286, + "141984615/train0": 0.35714285714285715, + "141984615/train1": 0.4642857142857143, + "141984615/train2": 0.14285714285714285, + "1461069713/train0": 0.5714285714285714, + "1461069713/train1": 0.5714285714285714, + "1461069713/train2": 0.6071428571428571, + "1480856810/train0": 0.35714285714285715, + "1480856810/train1": 0.39285714285714285, + "1480856810/train2": 0.6071428571428571, + "1509980664/train0": 0.64, + "1509980664/train1": 0.4, + "1509980664/train2": 0.5714285714285714, + "1522272686/train0": 0.14285714285714285, + "1522272686/train1": 0.2857142857142857, + "1522272686/train2": 0.6428571428571429, + "1549324805/train0": 0.75, + "1549324805/train1": 0.8571428571428571, + "1549324805/train2": 0.6071428571428571, + "1578990937/train0": 0.5714285714285714, + "1578990937/train1": 0.8928571428571429, + "1578990937/train2": 0.6071428571428571, + "1790188787/train0": 0.6785714285714286, + "1790188787/train1": 0.75, + "1790188787/train2": 0.6818181818181818, + "1822210471/train0": 0.5714285714285714, + "1822210471/train1": 0.42857142857142855, + "1822210471/train2": 0.25, + "1845489013/train0": 0.6785714285714286, + "1845489013/train1": 0.42857142857142855, + "1845489013/train2": 0.21428571428571427, + "1999022574/train0": 0.7142857142857143, + "1999022574/train1": 0.42857142857142855, + "1999022574/train2": 0.7142857142857143, + "2012211287/train0": 0.4642857142857143, + "2012211287/train1": 0.2222222222222222, + "2012211287/train2": 0.32142857142857145, + "2038594191/train0": 0.39285714285714285, + "2038594191/train1": 0.6428571428571429, + "2038594191/train2": 0.4642857142857143, + "2054899126/train0": 0.5714285714285714, + "2054899126/train1": 0.39285714285714285, + "2054899126/train2": 0.75, + "2061635888/train0": 0.6428571428571429, + "2061635888/train1": 0.7857142857142857, + "2061635888/train2": 0.75, + "211245842/train0": 0.6071428571428571, + "211245842/train1": 0.6428571428571429, + "211245842/train2": 0.6428571428571429, + "278008524/train0": 0.5, + "278008524/train1": 0.75, + "278008524/train2": 0.75, + "307068879/train0": 0.32142857142857145, + "307068879/train1": 0.4642857142857143, + "307068879/train2": 0.6071428571428571, + "412102563/train0": 0.42857142857142855, + "412102563/train1": 0.4642857142857143, + "412102563/train2": 0.42857142857142855, + "440577711/train0": 0.6428571428571429, + "440577711/train1": 0.0, + "441046872/train0": 0.5357142857142857, + "441046872/train1": 0.5714285714285714, + "441046872/train2": 0.6785714285714286, + "509430392/train0": 0.28, + "509430392/train1": 0.09090909090909091, + "509430392/train2": 0.09090909090909091, + "585904130/train0": 0.6785714285714286, + "585904130/train1": 0.35714285714285715, + "585904130/train2": 0.4642857142857143, + "608016290/train0": 0.39285714285714285, + "608016290/train1": 0.14285714285714285, + "608016290/train2": 0.10714285714285714, + "626486813/train0": 0.48214285714285715, + "626486813/train1": 0.5714285714285714, + "626486813/train2": 0.25, + "636680137/train0": 0.39285714285714285, + "636680137/train1": 0.39285714285714285, + "636680137/train2": 0.32142857142857145, + "655884605/train0": 0.6785714285714286, + "655884605/train1": 0.6071428571428571, + "655884605/train2": 0.21428571428571427, + "672328262/train0": 0.75, + "672328262/train1": 0.07142857142857142, + "672328262/train2": 0.5357142857142857, + "701667486/train0": 0.5714285714285714, + "701667486/train1": 0.5357142857142857, + "701667486/train2": 0.6071428571428571, + "747260437/train0": 0.39285714285714285, + "747260437/train1": 0.5357142857142857, + "747260437/train2": 0.8571428571428571, + "750448296/train0": 0.5357142857142857, + "750448296/train1": 0.5833333333333334, + "750448296/train2": 0.5833333333333334, + "767747424/train0": 0.45535714285714285, + "767747424/train1": 0.44642857142857145, + "767747424/train2": 0.4017857142857143 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5316972286972286, + "low": 0.37671318224259404, + "mean": 0.4473676018970137, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4473676018970137, + "per_task": { + "LiftPegUpright-v1": 0.3862433862433862, + "PickCube-v1": 0.35714285714285715, + "PullCube-v1": 0.5088661338661338, + "PushCube-v1": 0.3967934244404833, + "StackCube-v1": 0.5877922077922076 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5291027905207011, + "low": 0.46436263018352564, + "mean": 0.49696175576772594, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.518259211716223, + "low": 0.5152106312790322, + "mean": 0.5166882268906825, + "n": 116, + "num_samples": 1000 + }, + "mean": 0.5166882268906825, + "per_seed": { + "1073161778/train0": 0.5092992512408128, + "1073161778/train1": 0.506518118161019, + "1073161778/train2": 0.5082470514492481, + "1113371261/train0": 0.5120817381301231, + "1113371261/train1": 0.5114294633513058, + "1113371261/train2": 0.5208647930155366, + "1151410346/train0": 0.5084242760212154, + "1151410346/train1": 0.5116087729702029, + "1151410346/train2": 0.5093949827060259, + "1261930456/train0": 0.507207152590294, + "1261930456/train1": 0.509870803335365, + "1261930456/train2": 0.5205192822349104, + "1324133122/train0": 0.5156896062548352, + "1324133122/train1": 0.5077543727318965, + "1324133122/train2": 0.5106739762139129, + "1324642411/train0": 0.5213088389133544, + "1324642411/train1": 0.5212524000863837, + "1324642411/train2": 0.527095259436081, + "1375492786/train0": 0.5390208488024293, + "1375492786/train1": 0.5136001990069363, + "1375492786/train2": 0.5233704587833103, + "141984615/train0": 0.5150349488116895, + "141984615/train1": 0.5209787160422698, + "141984615/train2": 0.5266278793488347, + "1461069713/train0": 0.5120775991497745, + "1461069713/train1": 0.5159485485371296, + "1461069713/train2": 0.5323058563946664, + "1480856810/train0": 0.5131988287429138, + "1480856810/train1": 0.5316435268709924, + "1480856810/train2": 0.5203685487667845, + "1509980664/train0": 0.5069861149914309, + "1509980664/train1": 0.5077947588157862, + "1509980664/train2": 0.5080683702895407, + "1522272686/train0": 0.5308364411582983, + "1522272686/train1": 0.5263848235942014, + "1522272686/train2": 0.5271177549323166, + "1549324805/train0": 0.5465855152184887, + "1549324805/train1": 0.5489097792855695, + "1549324805/train2": 0.5156532908328474, + "1578990937/train0": 0.5120526217939588, + "1578990937/train1": 0.517049827852834, + "1578990937/train2": 0.5121887042635905, + "1790188787/train0": 0.5223174936773409, + "1790188787/train1": 0.5184372952384458, + "1790188787/train2": 0.5109241069247938, + "1822210471/train0": 0.5138796533820333, + "1822210471/train1": 0.5083982289146939, + "1822210471/train2": 0.5174993924353067, + "1845489013/train0": 0.5068891153127965, + "1845489013/train1": 0.5120112216374724, + "1845489013/train2": 0.5049839841564138, + "1999022574/train0": 0.5125355587394795, + "1999022574/train1": 0.5081622486924189, + "1999022574/train2": 0.5126369132288152, + "2012211287/train0": 0.5090347138321765, + "2012211287/train1": 0.5141496618096716, + "2012211287/train2": 0.5108011563830763, + "2038594191/train0": 0.5291333104927968, + "2038594191/train1": 0.5272185314816756, + "2038594191/train2": 0.5202926621233347, + "2054899126/train0": 0.5096744733127383, + "2054899126/train1": 0.5220705265713083, + "2054899126/train2": 0.5259225813730811, + "2061635888/train0": 0.5035728957399334, + "2061635888/train1": 0.5095019143713128, + "2061635888/train2": 0.5079491341690449, + "211245842/train0": 0.5129474725633024, + "211245842/train1": 0.5130796019472789, + "211245842/train2": 0.5205888094479262, + "278008524/train0": 0.5223926438814469, + "278008524/train1": 0.5243015603781166, + "278008524/train2": 0.514035350060809, + "307068879/train0": 0.5119848106472845, + "307068879/train1": 0.5075913402530835, + "307068879/train2": 0.50920166710305, + "412102563/train0": 0.5149085793100113, + "412102563/train1": 0.5120484995354441, + "412102563/train2": 0.5130901931270576, + "440577711/train0": 0.5163938105237615, + "440577711/train1": 0.5239216972771956, + "441046872/train0": 0.510976779082865, + "441046872/train1": 0.5088684124078007, + "441046872/train2": 0.5127281442153561, + "509430392/train0": 0.5167159497943329, + "509430392/train1": 0.5152174445598239, + "509430392/train2": 0.513267600062783, + "585904130/train0": 0.5316537240199258, + "585904130/train1": 0.5109791799445677, + "585904130/train2": 0.5257097365720064, + "608016290/train0": 0.5141263992957481, + "608016290/train1": 0.5114864515125813, + "608016290/train2": 0.5207029573796137, + "626486813/train0": 0.5206261667386757, + "626486813/train1": 0.517265264607039, + "626486813/train2": 0.5332777485074317, + "636680137/train0": 0.5172753273454199, + "636680137/train1": 0.521068070904193, + "636680137/train2": 0.5248912478073117, + "655884605/train0": 0.5125805300949516, + "655884605/train1": 0.506450790055417, + "655884605/train2": 0.505842678549458, + "672328262/train0": 0.5165144952070263, + "672328262/train1": 0.5183234734272898, + "672328262/train2": 0.5090400478771011, + "701667486/train0": 0.5106631871119919, + "701667486/train1": 0.5098775552568305, + "701667486/train2": 0.5194690186587149, + "747260437/train0": 0.5120181933719066, + "747260437/train1": 0.5088316663078858, + "747260437/train2": 0.5073520007934037, + "750448296/train0": 0.5233276622092005, + "750448296/train1": 0.5305656417959553, + "750448296/train2": 0.526807577707075, + "767747424/train0": 0.524268127667919, + "767747424/train1": 0.5309237590932049, + "767747424/train2": 0.5186143981913337 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5236891115071638, + "low": 0.5127247628737179, + "mean": 0.518166359167713, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.518166359167713, + "per_task": { + "LiftPegUpright-v1": 0.5110477943182259, + "PickCube-v1": 0.5281130065616054, + "PullCube-v1": 0.522002123999376, + "PushCube-v1": 0.5182152964138562, + "StackCube-v1": 0.5114535745455024 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5196724223172199, + "low": 0.516156628694114, + "mean": 0.51783450833921, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.17289303773014583, + "low": 0.13439010370804003, + "mean": 0.15270185904199562, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.15270185904199562, + "per_seed": { + "1073161778/train0": 0.15215639409795562, + "1073161778/train1": 0.11366097530387614, + "1073161778/train2": 0.2939613371635338, + "1113371261/train0": 0.04779602384440884, + "1113371261/train1": 0.1671419652201228, + "1113371261/train2": 0.05056377841303483, + "1151410346/train0": 0.20586143826449887, + "1151410346/train1": 0.047323058684488606, + "1151410346/train2": 0.0977478744368312, + "1261930456/train0": 0.00720715259029403, + "1261930456/train1": 0.025843482378920646, + "1261930456/train2": 0.12766213937776755, + "1324133122/train0": 0.05573896517373622, + "1324133122/train1": 0.2422456272681035, + "1324133122/train2": 0.08210254764248431, + "1324642411/train0": 0.1284516960562116, + "1324642411/train1": 0.33606721490119856, + "1324642411/train2": 0.3604285927694144, + "1375492786/train0": 0.153286843505263, + "1375492786/train1": 0.3078283724216351, + "1375492786/train2": 0.15520096978811826, + "141984615/train0": 0.1578920916688324, + "141984615/train1": 0.05669300175655545, + "141984615/train2": 0.38377073649169186, + "1461069713/train0": 0.05935097227879693, + "1461069713/train1": 0.05548002289144183, + "1461069713/train2": 0.07483700074819066, + "1480856810/train0": 0.15605597160005663, + "1480856810/train1": 0.13878638401384952, + "1480856810/train2": 0.08677430837607258, + "1509980664/train0": 0.13301388500856914, + "1509980664/train1": 0.10779475881578615, + "1509980664/train2": 0.06336020113903074, + "1522272686/train0": 0.38797929830115546, + "1522272686/train1": 0.24067053787991566, + "1522272686/train2": 0.1157393879248263, + "1549324805/train0": 0.20341448478151117, + "1549324805/train1": 0.3082330778572876, + "1549324805/train2": 0.0914895663100097, + "1578990937/train0": 0.05937594963461257, + "1578990937/train1": 0.37580731500430886, + "1578990937/train2": 0.09495415287926656, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.15625393489408768, + "1790188787/train1": 0.23156270476155416, + "1790188787/train2": 0.17089407489338793, + "1822210471/train0": 0.057548918046538056, + "1822210471/train1": 0.07982680034326534, + "1822210471/train2": 0.26749939243530674, + "1845489013/train0": 0.17168231325863215, + "1845489013/train1": 0.08343979306604382, + "1845489013/train2": 0.29069826987069947, + "1999022574/train0": 0.20175015554623477, + "1999022574/train1": 0.07959082012099034, + "1999022574/train2": 0.20164880105689909, + "2012211287/train0": 0.044748999546462165, + "2012211287/train1": 0.2919274395874494, + "2012211287/train2": 0.18937258495450487, + "2038594191/train0": 0.13627616763565392, + "2038594191/train1": 0.11563861137546727, + "2038594191/train2": 0.05600694783762039, + "2054899126/train0": 0.03087704905791655, + "2054899126/train1": 0.0646066918570827, + "2054899126/train2": 0.11203870931345944, + "2061635888/train0": 0.13928424711720955, + "2061635888/train1": 0.27621237134297294, + "2061635888/train2": 0.24205086583095514, + "211245842/train0": 0.09419538457955468, + "211245842/train1": 0.12977754090986404, + "211245842/train2": 0.12226833340921672, + "278008524/train0": 0.011196321940723453, + "278008524/train1": 0.1128492198109417, + "278008524/train2": 0.11798232496959549, + "307068879/train0": 0.19055623921871306, + "307068879/train1": 0.0433056259673692, + "307068879/train2": 0.09794119003980706, + "412102563/train0": 0.08633715073858278, + "412102563/train1": 0.04776278524972977, + "412102563/train2": 0.08451876455562907, + "440577711/train0": 0.12646333233338136, + "440577711/train1": 0.5239216972771956, + "440577711/train2": 0.0, + "441046872/train0": 0.024737506631420714, + "441046872/train1": 0.06256015902077072, + "441046872/train2": 0.16584328435607254, + "509430392/train0": 0.23671594979433286, + "509430392/train1": 0.42430835365073294, + "509430392/train2": 0.4223585091536921, + "585904130/train0": 0.14691770455150277, + "585904130/train1": 0.1538363228017105, + "585904130/train2": 0.06142402228629207, + "608016290/train0": 0.12126925643860526, + "608016290/train1": 0.36862930865543847, + "608016290/train2": 0.41356010023675654, + "626486813/train0": 0.26398015577572026, + "626486813/train1": 0.0748773088696959, + "626486813/train2": 0.2832777485074317, + "636680137/train0": 0.12441818448827707, + "636680137/train1": 0.1282109280470502, + "636680137/train2": 0.20346267637874021, + "655884605/train0": 0.16599089847647697, + "655884605/train1": 0.10069206708744005, + "655884605/train2": 0.29155696426374367, + "672328262/train0": 0.2334855047929737, + "672328262/train1": 0.4468949019987184, + "672328262/train2": 0.02667423783718459, + "701667486/train0": 0.06076538431657952, + "701667486/train1": 0.02583673045745516, + "701667486/train2": 0.08767383848414223, + "747260437/train0": 0.11916105051476372, + "747260437/train1": 0.02688261940639991, + "747260437/train2": 0.3497908563494534, + "750448296/train0": 0.104589703577334, + "750448296/train1": 0.09714266836899327, + "750448296/train2": 0.05652575562625828, + "767747424/train0": 0.1905210480971844, + "767747424/train1": 0.12149106073317761, + "767747424/train2": 0.14789818559258952 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.21632856809992243, + "low": 0.13963892944806727, + "mean": 0.17727089899279783, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.17727089899279783, + "per_task": { + "LiftPegUpright-v1": 0.17161536008124903, + "PickCube-v1": 0.24812974136863244, + "PullCube-v1": 0.13393386645915098, + "PushCube-v1": 0.20331974993456392, + "StackCube-v1": 0.12935577712039273 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.1665179621010634, + "low": 0.130876290606077, + "mean": 0.14786219273302828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 26.716666666666665, + "low": 24.608333333333334, + "mean": 25.747916666666665, + "n": 120, + "num_samples": 1000 + }, + "mean": 25.747916666666665, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 28.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 27.0, + "1324642411/train2": 18.0, + "1375492786/train0": 13.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 28.0, + "1480856810/train1": 28.0, + "1480856810/train2": 28.0, + "1509980664/train0": 25.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 28.0, + "1790188787/train1": 28.0, + "1790188787/train2": 22.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 28.0, + "1999022574/train1": 28.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 27.0, + "2012211287/train2": 28.0, + "2038594191/train0": 28.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 14.0, + "278008524/train1": 14.0, + "278008524/train2": 14.0, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 28.0, + "412102563/train1": 28.0, + "412102563/train2": 28.0, + "440577711/train0": 28.0, + "440577711/train1": 13.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 25.0, + "509430392/train1": 22.0, + "509430392/train2": 22.0, + "585904130/train0": 28.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 28.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 28.0, + "672328262/train1": 28.0, + "672328262/train2": 28.0, + "701667486/train0": 28.0, + "701667486/train1": 28.0, + "701667486/train2": 28.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 28.0, + "750448296/train1": 28.0, + "750448296/train2": 28.0, + "767747424/train0": 22.75, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 27.927272727272726, + "low": 24.69563636363636, + "mean": 26.348969696969696, + "n": 5, + "num_samples": 1000 + }, + "mean": 26.348969696969696, + "per_task": { + "LiftPegUpright-v1": 27.933333333333334, + "PickCube-v1": 28.0, + "PullCube-v1": 24.16, + "PushCube-v1": 23.833333333333332, + "StackCube-v1": 27.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 26.604166666666668, + "low": 24.104166666666668, + "mean": 25.430555555555557, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.27383652765997796, + "low": 0.19809712028868312, + "mean": 0.23349326650515886, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23349326650515886, + "per_seed": { + "1073161778/train0": 0.20584099091859903, + "1073161778/train1": 0.19554991093790747, + "1073161778/train2": 0.19441156679128385, + "1113371261/train0": 0.11841214667429321, + "1113371261/train1": 0.111487318938856, + "1113371261/train2": 0.1637967593188533, + "1151410346/train0": 0.14682773551354072, + "1151410346/train1": 0.1505309920780672, + "1151410346/train2": 0.15717334321041485, + "1261930456/train0": 0.006272514778440541, + "1261930456/train1": 0.077102113380139, + "1261930456/train2": 0.09402188754699127, + "1324133122/train0": 0.07612377904669673, + "1324133122/train1": 0.21872963666248138, + "1324133122/train2": 0.0939715257270842, + "1324642411/train0": 0.8928362396780647, + "1324642411/train1": 0.7531487720723857, + "1324642411/train2": 0.6491618422591205, + "1375492786/train0": 0.23249138223608884, + "1375492786/train1": -0.037632419519960006, + "1375492786/train2": 0.17970484878502413, + "141984615/train0": 0.18927476177277072, + "141984615/train1": 0.18318239803192132, + "141984615/train2": 0.114212135741804, + "1461069713/train0": 0.21845740942913472, + "1461069713/train1": 0.30554439462118227, + "1461069713/train2": 0.5338556706581634, + "1480856810/train0": 0.7158970211825535, + "1480856810/train1": 0.7444206849830362, + "1480856810/train2": 0.5127510371288293, + "1509980664/train0": 0.1645139076256539, + "1509980664/train1": 0.12250900335942658, + "1509980664/train2": 0.1805548023297982, + "1522272686/train0": 0.4948526502859529, + "1522272686/train1": 0.3317013229350659, + "1522272686/train2": 0.3719790973396491, + "1549324805/train0": 0.4099932021305159, + "1549324805/train1": 0.4366493356305453, + "1549324805/train2": 0.33492807900387084, + "1578990937/train0": 0.12075832280916365, + "1578990937/train1": 0.15613350902346218, + "1578990937/train2": 0.14872603579685487, + "1747178030/train0": 0.6873128160326993, + "1747178030/train1": 0.9835802031786316, + "1747178030/train2": 0.9536859431190323, + "1790188787/train0": 0.25334075772729825, + "1790188787/train1": 0.27780618403492885, + "1790188787/train2": 0.18703346552659444, + "1822210471/train0": 0.0577847490536553, + "1822210471/train1": -0.07212192716463756, + "1822210471/train2": 0.0031507691650567438, + "1845489013/train0": 0.05982643176183276, + "1845489013/train1": 0.0787154239384655, + "1845489013/train2": 0.003957935292008923, + "1999022574/train0": 0.19337117648183325, + "1999022574/train1": 0.09814524693113953, + "1999022574/train2": 0.1565539695903909, + "2012211287/train0": 0.18515916980019373, + "2012211287/train1": 0.13504095940734823, + "2012211287/train2": 0.12797764508486972, + "2038594191/train0": 0.07343156967546631, + "2038594191/train1": 0.11157849082379764, + "2038594191/train2": 0.04359967653884958, + "2054899126/train0": 0.38208593853249617, + "2054899126/train1": 0.5828384224098009, + "2054899126/train2": 0.4502950976842904, + "2061635888/train0": -0.08547837727895316, + "2061635888/train1": 0.03930678750007065, + "2061635888/train2": 0.008004389977088786, + "211245842/train0": 0.22532734972882426, + "211245842/train1": 0.22424307526810303, + "211245842/train2": 0.2206154553304457, + "278008524/train0": 0.215222220593497, + "278008524/train1": 0.1986640133517516, + "278008524/train2": 0.252213252965857, + "307068879/train0": -0.005683666382895997, + "307068879/train1": -0.01192475167673359, + "307068879/train2": -0.003886783551851819, + "412102563/train0": 0.22150303258309284, + "412102563/train1": 0.12051079262473083, + "412102563/train2": 0.15822655084813653, + "440577711/train0": 0.5586790894555644, + "440577711/train1": 0.660329471290032, + "440577711/train2": 0.6996994006285734, + "441046872/train0": 0.006895278681057632, + "441046872/train1": -0.03548490283103705, + "441046872/train2": 0.0013491370718414428, + "509430392/train0": 0.40670225613190863, + "509430392/train1": 0.46272796115382003, + "509430392/train2": 0.4020750348567389, + "585904130/train0": 0.15837838927275857, + "585904130/train1": 0.07464422139595693, + "585904130/train2": 0.12634263058484524, + "608016290/train0": 0.2141408309665893, + "608016290/train1": 0.19953791912822288, + "608016290/train2": 0.24661642979812717, + "626486813/train0": 0.2289130081988947, + "626486813/train1": 0.2919681543477006, + "626486813/train2": 0.29076199506255523, + "636680137/train0": 0.19058677054848244, + "636680137/train1": 0.12268453237476608, + "636680137/train2": 0.3268104640308388, + "655884605/train0": 0.03266312735678528, + "655884605/train1": 0.03338313576240709, + "655884605/train2": 0.07788535796650367, + "672328262/train0": 0.02720919346098577, + "672328262/train1": 0.047096220474437275, + "672328262/train2": -0.0702178669924084, + "701667486/train0": 0.18005163348657682, + "701667486/train1": 0.12475135370966416, + "701667486/train2": 0.2642366239473226, + "747260437/train0": 0.1795266808160827, + "747260437/train1": 0.17198143779324082, + "747260437/train2": 0.1459627586815193, + "750448296/train0": 0.3509091319089219, + "750448296/train1": 0.3670406155313691, + "750448296/train2": 0.35168270720155403, + "767747424/train0": 0.3467289216925133, + "767747424/train1": 0.269433473428486, + "767747424/train2": 0.2226382768839351 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3856920812983886, + "low": 0.15092968931612322, + "mean": 0.27477092592087266, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.27477092592087266, + "per_task": { + "LiftPegUpright-v1": 0.11036931560956814, + "PickCube-v1": 0.3995110235202226, + "PullCube-v1": 0.26581590627488516, + "PushCube-v1": 0.46411142965639, + "StackCube-v1": 0.13404695454329726 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.28294360754900316, + "low": 0.20997846528177866, + "mean": 0.24699875700739174, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.29583333333333334, + "low": 0.15416666666666667, + "mean": 0.22291666666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.22291666666666668, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40606060606060607, + "low": 0.15903030303030302, + "mean": 0.2754747474747475, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2754747474747475, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.3333333333333333, + "PullCube-v1": 0.17333333333333334, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.18181818181818182 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.2916666666666667, + "low": 0.1597222222222222, + "mean": 0.2222222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7715499536289523, + "low": 0.5113214392965246, + "mean": 0.6364790620334033, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6364790620334033, + "per_seed": { + "1073161778/train0": 0.7180102467536926, + "1073161778/train1": 0.7134289145469666, + "1073161778/train2": 2.0, + "1113371261/train0": 0.0740400105714798, + "1113371261/train1": 0.09622243046760559, + "1113371261/train2": 0.1003647893667221, + "1151410346/train0": 0.5684944987297058, + "1151410346/train1": 0.2439272701740265, + "1151410346/train2": 0.5630103945732117, + "1261930456/train0": 0.09678153693675995, + "1261930456/train1": 0.09997129440307617, + "1261930456/train2": 0.08545441925525665, + "1324133122/train0": 0.0628519058227539, + "1324133122/train1": 0.0843733698129654, + "1324133122/train2": 0.09815146028995514, + "1324642411/train0": 0.21000581979751587, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.14743654429912567, + "1375492786/train2": 0.19452734291553497, + "141984615/train0": 0.023719489574432373, + "141984615/train1": 0.04397805407643318, + "141984615/train2": 0.04447716474533081, + "1461069713/train0": 0.05616961047053337, + "1461069713/train1": 0.05644957348704338, + "1461069713/train2": 0.11542954295873642, + "1480856810/train0": 0.14178013801574707, + "1480856810/train1": 2.0, + "1480856810/train2": 0.19622710347175598, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.83889240026474, + "1522272686/train0": 0.5233199596405029, + "1522272686/train1": 0.5422300696372986, + "1522272686/train2": 2.0, + "1549324805/train0": 0.21791090071201324, + "1549324805/train1": 0.3088321089744568, + "1549324805/train2": 0.21403594315052032, + "1578990937/train0": 0.21425709128379822, + "1578990937/train1": 0.21763037145137787, + "1578990937/train2": 0.23259200155735016, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.1862330436706543, + "1790188787/train2": 2.0, + "1822210471/train0": 0.03802649304270744, + "1822210471/train1": 0.07539409399032593, + "1822210471/train2": 0.058819372206926346, + "1845489013/train0": 0.0428844690322876, + "1845489013/train1": 0.09529292583465576, + "1845489013/train2": 0.0861951932311058, + "1999022574/train0": 0.5970457196235657, + "1999022574/train1": 2.0, + "1999022574/train2": 0.7970014810562134, + "2012211287/train0": 0.7277753949165344, + "2012211287/train1": 2.0, + "2012211287/train2": 0.7270426154136658, + "2038594191/train0": 0.15247176587581635, + "2038594191/train1": 0.1902807205915451, + "2038594191/train2": 0.24880830943584442, + "2054899126/train0": 1.0726270377635956, + "2054899126/train1": 1.0755530372262, + "2054899126/train2": 1.0683744624257088, + "2061635888/train0": 0.07642464339733124, + "2061635888/train1": 0.09046469628810883, + "2061635888/train2": 0.07902413606643677, + "211245842/train0": 0.5822281837463379, + "211245842/train1": 0.2016763538122177, + "211245842/train2": 0.2142850160598755, + "278008524/train0": 1.0722030997276306, + "278008524/train1": 1.1246033310890198, + "278008524/train2": 1.157032236456871, + "307068879/train0": 0.20280277729034424, + "307068879/train1": 0.20222051441669464, + "307068879/train2": 0.2035302072763443, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 0.8275061845779419, + "440577711/train0": 0.2057151049375534, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.203963503241539, + "441046872/train1": 0.20609354972839355, + "441046872/train2": 0.2042645812034607, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 0.16448359191417694, + "585904130/train1": 0.15698926150798798, + "585904130/train2": 0.2938849627971649, + "608016290/train0": 0.6124615669250488, + "608016290/train1": 0.6101007461547852, + "608016290/train2": 0.5705108642578125, + "626486813/train0": 0.05774742551147938, + "626486813/train1": 0.0619367565959692, + "626486813/train2": 0.08006523177027702, + "636680137/train0": 0.10778365284204483, + "636680137/train1": 0.11253269761800766, + "636680137/train2": 0.1247115358710289, + "655884605/train0": 0.15739890933036804, + "655884605/train1": 0.15038728713989258, + "655884605/train2": 0.17242512106895447, + "672328262/train0": 0.05979333445429802, + "672328262/train1": 0.053388457745313644, + "672328262/train2": 0.07216622680425644, + "701667486/train0": 0.7879213094711304, + "701667486/train1": 0.7947455048561096, + "701667486/train2": 2.0, + "747260437/train0": 0.19230616092681885, + "747260437/train1": 0.1342230886220932, + "747260437/train2": 0.18527252972126007, + "750448296/train0": 0.20943730572859445, + "750448296/train1": 0.22532296180725098, + "750448296/train2": 0.2481476068496704, + "767747424/train0": 0.5789013803005219, + "767747424/train1": 0.5825987188145518, + "767747424/train2": 0.5809952197596431 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.021850009759267, + "low": 0.5703265610120514, + "mean": 0.7913498574552512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7913498574552512, + "per_task": { + "LiftPegUpright-v1": 0.6601470321416855, + "PickCube-v1": 1.021850009759267, + "PullCube-v1": 0.46170220340291657, + "PushCube-v1": 1.1790093589160178, + "StackCube-v1": 0.634040683056369 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 0.7532538059684966, + "low": 0.5049100180880891, + "mean": 0.6232009150553495, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 4.0625, + "low": 3.2270833333333333, + "mean": 3.6576388888888887, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.6576388888888887, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 5.0, + "1073161778/train2": 7.0, + "1113371261/train0": 7.0, + "1113371261/train1": 3.0, + "1113371261/train2": 6.0, + "1151410346/train0": 5.0, + "1151410346/train1": 4.0, + "1151410346/train2": 7.0, + "1261930456/train0": 3.0, + "1261930456/train1": 3.0, + "1261930456/train2": 7.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 6.0, + "1324642411/train0": 1.0, + "1324642411/train1": 3.0, + "1324642411/train2": 6.0, + "1375492786/train0": 6.0, + "1375492786/train1": 4.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 6.0, + "1461069713/train0": 1.0, + "1461069713/train1": 6.0, + "1461069713/train2": 7.0, + "1480856810/train0": 6.0, + "1480856810/train1": 0.0, + "1480856810/train2": 7.0, + "1509980664/train0": 6.0, + "1509980664/train1": 2.0, + "1509980664/train2": 5.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 4.0, + "1578990937/train0": 3.0, + "1578990937/train1": 5.0, + "1578990937/train2": 5.0, + "1747178030/train0": 2.0, + "1747178030/train1": 3.0, + "1747178030/train2": 6.0, + "1790188787/train0": 4.0, + "1790188787/train1": 2.0, + "1790188787/train2": 6.0, + "1822210471/train0": 1.0, + "1822210471/train1": 5.0, + "1822210471/train2": 7.0, + "1845489013/train0": 3.0, + "1845489013/train1": 4.0, + "1845489013/train2": 2.0, + "1999022574/train0": 7.0, + "1999022574/train1": 0.0, + "1999022574/train2": 3.0, + "2012211287/train0": 2.0, + "2012211287/train1": 7.0, + "2012211287/train2": 5.0, + "2038594191/train0": 4.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 3.0, + "2054899126/train2": 1.0, + "2061635888/train0": 2.0, + "2061635888/train1": 3.0, + "2061635888/train2": 0.0, + "211245842/train0": 7.0, + "211245842/train1": 3.0, + "211245842/train2": 1.0, + "278008524/train0": 4.5, + "278008524/train1": 1.0, + "278008524/train2": 3.0, + "307068879/train0": 6.0, + "307068879/train1": 3.0, + "307068879/train2": 0.0, + "412102563/train0": 6.0, + "412102563/train1": 3.0, + "412102563/train2": 2.0, + "440577711/train0": 7.0, + "440577711/train1": 7.0, + "440577711/train2": 7.0, + "441046872/train0": 7.0, + "441046872/train1": 3.0, + "441046872/train2": 6.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 5.0, + "585904130/train0": 6.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 7.0, + "608016290/train1": 1.0, + "608016290/train2": 6.0, + "626486813/train0": 1.0, + "626486813/train1": 1.5, + "626486813/train2": 2.5, + "636680137/train0": 4.0, + "636680137/train1": 3.0, + "636680137/train2": 7.0, + "655884605/train0": 6.0, + "655884605/train1": 4.0, + "655884605/train2": 7.0, + "672328262/train0": 0.0, + "672328262/train1": 6.0, + "672328262/train2": 3.0, + "701667486/train0": 4.0, + "701667486/train1": 6.0, + "701667486/train2": 5.0, + "747260437/train0": 6.0, + "747260437/train1": 0.0, + "747260437/train2": 5.0, + "750448296/train0": 2.3333333333333335, + "750448296/train1": 2.3333333333333335, + "750448296/train2": 3.0, + "767747424/train0": 4.0, + "767747424/train1": 2.75, + "767747424/train2": 3.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 4.387272727272728, + "low": 2.261333333333333, + "mean": 3.452969696969697, + "n": 5, + "num_samples": 1000 + }, + "mean": 3.452969696969697, + "per_task": { + "LiftPegUpright-v1": 4.733333333333333, + "PickCube-v1": 1.3333333333333333, + "PullCube-v1": 2.88, + "PushCube-v1": 4.166666666666667, + "StackCube-v1": 4.151515151515151 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.8541666666666665, + "low": 3.1180555555555554, + "mean": 3.4930555555555554, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.325, + "low": 0.17777777777777773, + "mean": 0.24999999999999997, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.24999999999999997, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7, + "low": 0.10303030303030303, + "mean": 0.38484848484848483, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.38484848484848483, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.5, + "StackCube-v1": 0.09090909090909091 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3472222222222222, + "low": 0.20833333333333334, + "mean": 0.2777777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.020833333333333332, + "low": -0.10416666666666667, + "mean": -0.058333333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.058333333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -1.0, + "1324642411/train1": -1.0, + "1324642411/train2": -1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": -1.0, + "1480856810/train1": -1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": -1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.2053333333333333, + "mean": -0.072, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.072, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.02666666666666667, + "PushCube-v1": -0.3333333333333333, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": -0.020833333333333332, + "low": -0.09722222222222222, + "mean": -0.05555555555555555, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8393616240057681, + "low": 0.5681936280801893, + "mean": 0.6955969328992068, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6955969328992068, + "per_seed": { + "1073161778/train0": 0.725517988204956, + "1073161778/train1": 0.725517988204956, + "1073161778/train2": 0.725517988204956, + "1113371261/train0": 0.11109928786754608, + "1113371261/train1": 0.11109928786754608, + "1113371261/train2": 0.11109928786754608, + "1151410346/train0": 0.2284735143184662, + "1151410346/train1": 0.2284735143184662, + "1151410346/train2": 0.2284735143184662, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.084227055311203, + "1324133122/train1": 0.084227055311203, + "1324133122/train2": 0.084227055311203, + "1324642411/train0": 0.14966940879821777, + "1324642411/train1": 0.17775490880012512, + "1324642411/train2": 0.14906781911849976, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.16783273220062256, + "141984615/train1": 0.16783273220062256, + "141984615/train2": 0.16783273220062256, + "1461069713/train0": 0.22852759063243866, + "1461069713/train1": 0.22852759063243866, + "1461069713/train2": 0.22852759063243866, + "1480856810/train0": 0.12667706608772278, + "1480856810/train1": 0.184015691280365, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8320798277854919, + "1509980664/train1": 0.8320798277854919, + "1509980664/train2": 0.8320798277854919, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.17249532043933868, + "1549324805/train1": 0.17249532043933868, + "1549324805/train2": 0.17249532043933868, + "1578990937/train0": 0.21426597237586975, + "1578990937/train1": 0.21426597237586975, + "1578990937/train2": 0.21426597237586975, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 0.09442546218633652, + "1822210471/train1": 0.09442546218633652, + "1822210471/train2": 0.09442546218633652, + "1845489013/train0": 0.11255800724029541, + "1845489013/train1": 0.11255800724029541, + "1845489013/train2": 0.11255800724029541, + "1999022574/train0": 0.8279688954353333, + "1999022574/train1": 0.8279688954353333, + "1999022574/train2": 0.8279688954353333, + "2012211287/train0": 0.7282466888427734, + "2012211287/train1": 0.7282466888427734, + "2012211287/train2": 0.7282466888427734, + "2038594191/train0": 0.09504059702157974, + "2038594191/train1": 0.09504059702157974, + "2038594191/train2": 0.09504059702157974, + "2054899126/train0": 2.0, + "2054899126/train1": 1.0613084435462952, + "2054899126/train2": 1.0612773001194, + "2061635888/train0": 0.08620692789554596, + "2061635888/train1": 0.08620692789554596, + "2061635888/train2": 0.08620692789554596, + "211245842/train0": 0.18430444598197937, + "211245842/train1": 0.18430444598197937, + "211245842/train2": 0.18430444598197937, + "278008524/train0": 1.1250072121620178, + "278008524/train1": 1.1250072121620178, + "278008524/train2": 1.1250072121620178, + "307068879/train0": 0.20915235579013824, + "307068879/train1": 0.20915235579013824, + "307068879/train2": 0.20915235579013824, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 0.21395862102508545, + "440577711/train2": 2.0, + "441046872/train0": 0.2097676396369934, + "441046872/train1": 0.2097676396369934, + "441046872/train2": 0.2097676396369934, + "509430392/train0": 0.04073460400104523, + "509430392/train1": 0.04073460400104523, + "509430392/train2": 0.04073460400104523, + "585904130/train0": 0.07103653997182846, + "585904130/train1": 0.07103653997182846, + "585904130/train2": 0.07103653997182846, + "608016290/train0": 0.525360107421875, + "608016290/train1": 0.525360107421875, + "608016290/train2": 0.525360107421875, + "626486813/train0": 0.04174089431762695, + "626486813/train1": 0.04174089431762695, + "626486813/train2": 0.04174089431762695, + "636680137/train0": 0.11957359313964844, + "636680137/train1": 0.11957359313964844, + "636680137/train2": 0.11957359313964844, + "655884605/train0": 0.1440753936767578, + "655884605/train1": 0.1440753936767578, + "655884605/train2": 0.1440753936767578, + "672328262/train0": 0.10593423992395401, + "672328262/train1": 0.10593423992395401, + "672328262/train2": 0.10593423992395401, + "701667486/train0": 0.7912740707397461, + "701667486/train1": 0.7912740707397461, + "701667486/train2": 0.7912740707397461, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7605033268531164, + "750448296/train1": 0.7605033268531164, + "750448296/train2": 0.7605033268531164, + "767747424/train0": 1.0861943028867245, + "767747424/train1": 1.0747087560594082, + "767747424/train2": 1.0747087560594082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5084034281637932, + "low": 0.54274790094838, + "mean": 0.9592354668770173, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9592354668770173, + "per_task": { + "LiftPegUpright-v1": 0.4796089559793472, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7537987943490346, + "PushCube-v1": 1.0624081848396196, + "StackCube-v1": 0.5003613992170854 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8632289723286198, + "low": 0.6021636158952283, + "mean": 0.731696648730172, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.03824081445733706, + "low": -0.19171596641341845, + "mean": -0.10720654328664143, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.10720654328664143, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -1.8503305912017822, + "1324642411/train1": -1.8222450911998749, + "1324642411/train2": -1.8509321808815002, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": -1.8733229339122772, + "1480856810/train1": -1.815984308719635, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": -0.9386915564537048, + "2054899126/train2": -0.9387226998806, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": -1.7860413789749146, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.011485546827316284, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.37651891303062435, + "mean": -0.1320998800330692, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.1320998800330692, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.049451817671457926, + "PushCube-v1": -0.611047582493888, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": -0.03767612990405825, + "low": -0.17914260427157083, + "mean": -0.10213710284895366, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.289573314304774, + "low": 0.10738638407136832, + "mean": 0.19405842884961103, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.19405842884961103, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.274482011795044, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.3400209844112396, + "1151410346/train1": 0.015453755855560303, + "1151410346/train2": 0.3345368802547455, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.00014631450176239014, + "1324133122/train2": 0.013924404978752136, + "1324642411/train0": 0.060336410999298096, + "1324642411/train1": 1.8222450911998749, + "1324642411/train2": 1.8509321808815002, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.015103071928024292, + "1480856810/train1": 1.815984308719635, + "1480856810/train2": 0.0, + "1509980664/train0": 1.167920172214508, + "1509980664/train1": 1.167920172214508, + "1509980664/train2": 0.006812572479248047, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.04541558027267456, + "1549324805/train1": 0.1363367885351181, + "1549324805/train2": 0.04154062271118164, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0033643990755081177, + "1578990937/train2": 0.018326029181480408, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.1720311045646667, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.2717533111572266, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0574311688542366, + "2038594191/train1": 0.09524012356996536, + "2038594191/train2": 0.15376771241426468, + "2054899126/train0": 0.0, + "2054899126/train1": 0.014244593679904938, + "2054899126/train2": 0.007097162306308746, + "2061635888/train0": 0.0, + "2061635888/train1": 0.004257768392562866, + "2061635888/train2": 0.0, + "211245842/train0": 0.3979237377643585, + "211245842/train1": 0.017371907830238342, + "211245842/train2": 0.029980570077896118, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.03202502429485321, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.7860413789749146, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.9592653959989548, + "509430392/train1": 1.9592653959989548, + "509430392/train2": 1.9592653959989548, + "585904130/train0": 0.09344705194234848, + "585904130/train1": 0.08595272153615952, + "585904130/train2": 0.22284842282533646, + "608016290/train0": 0.08710145950317383, + "608016290/train1": 0.08474063873291016, + "608016290/train2": 0.0451507568359375, + "626486813/train0": 0.016006531193852425, + "626486813/train1": 0.020195862278342247, + "626486813/train2": 0.03832433745265007, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.005137942731380463, + "655884605/train0": 0.01332351565361023, + "655884605/train1": 0.006311893463134766, + "655884605/train2": 0.028349727392196655, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0034714341163635254, + "701667486/train2": 1.208725929260254, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.06344164411226909, + "750448296/train1": 0.06390197078386943, + "750448296/train2": 0.11175449689229329, + "767747424/train0": 0.007970128208398819, + "767747424/train1": 0.019207116216421127, + "767747424/train2": 0.013884376734495163 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4825414665016261, + "low": 0.04237489831447601, + "mean": 0.22541920597083642, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.22541920597083642, + "per_task": { + "LiftPegUpright-v1": 0.18421521186828613, + "PickCube-v1": 0.0, + "PullCube-v1": 0.027659279704093934, + "PushCube-v1": 0.7349132572611173, + "StackCube-v1": 0.18030828102068466 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.2424813198287868, + "low": 0.09229205804876983, + "mean": 0.16677976430704197, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.16666666666666666, + "low": 0.058333333333333334, + "mean": 0.10833333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.10833333333333334, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.26, + "low": 0.024242424242424242, + "mean": 0.12868686868686868, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.12868686868686868, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.1388888888888889, + "low": 0.041666666666666664, + "mean": 0.09027777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6597222222222222, + "low": 0.4847222222222222, + "mean": 0.5708333333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5708333333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8072727272727273, + "low": 0.40533333333333327, + "mean": 0.6063030303030303, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6063030303030303, + "per_task": { + "LiftPegUpright-v1": 0.8666666666666667, + "PickCube-v1": 0.6666666666666666, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.8181818181818182 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.5972222222222222, + "low": 0.4375, + "mean": 0.5138888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1496533814558965, + "low": 0.8442967605000982, + "mean": 0.9982878092624661, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.9982878092624661, + "per_seed": { + "1073161778/train0": 1.2819897532463074, + "1073161778/train1": 1.2865710854530334, + "1073161778/train2": 0.0, + "1113371261/train0": 1.9259599894285202, + "1113371261/train1": 1.9037775695323944, + "1113371261/train2": 1.899635210633278, + "1151410346/train0": 1.4315055012702942, + "1151410346/train1": 1.7560727298259735, + "1151410346/train2": 1.4369896054267883, + "1261930456/train0": 1.90321846306324, + "1261930456/train1": 1.9000287055969238, + "1261930456/train2": 1.9145455807447433, + "1324133122/train0": 1.937148094177246, + "1324133122/train1": 1.9156266301870346, + "1324133122/train2": 1.9018485397100449, + "1324642411/train0": 1.7899941802024841, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.8525634557008743, + "1375492786/train2": 1.805472657084465, + "141984615/train0": 0.2620577812194824, + "141984615/train1": 0.2417992167174816, + "141984615/train2": 0.24130010604858398, + "1461069713/train0": 0.10317284241318703, + "1461069713/train1": 0.10289287939667702, + "1461069713/train2": 0.04391290992498398, + "1480856810/train0": 0.06770022213459015, + "1480856810/train1": 0.0, + "1480856810/train2": 0.013253256678581238, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.16110759973526, + "1522272686/train0": 1.476680040359497, + "1522272686/train1": 1.4577699303627014, + "1522272686/train2": 0.0, + "1549324805/train0": 0.07954518496990204, + "1549324805/train1": 0.0, + "1549324805/train2": 0.08342014253139496, + "1578990937/train0": 1.7857429087162018, + "1578990937/train1": 1.7823696285486221, + "1578990937/train2": 1.7674079984426498, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8137669563293457, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9619735069572926, + "1822210471/train1": 1.924605906009674, + "1822210471/train2": 1.9411806277930737, + "1845489013/train0": 1.9571155309677124, + "1845489013/train1": 1.9047070741653442, + "1845489013/train2": 1.9138048067688942, + "1999022574/train0": 1.4029542803764343, + "1999022574/train1": 0.0, + "1999022574/train2": 1.2029985189437866, + "2012211287/train0": 1.2722246050834656, + "2012211287/train1": 0.0, + "2012211287/train2": 1.2729573845863342, + "2038594191/train0": 1.8475282341241837, + "2038594191/train1": 1.809719279408455, + "2038594191/train2": 1.7511916905641556, + "2054899126/train0": 0.9273729622364044, + "2054899126/train1": 0.9244469627737999, + "2054899126/train2": 0.9316255375742912, + "2061635888/train0": 1.9235753566026688, + "2061635888/train1": 1.9095353037118912, + "2061635888/train2": 1.9209758639335632, + "211245842/train0": 1.417771816253662, + "211245842/train1": 1.7983236461877823, + "211245842/train2": 1.7857149839401245, + "278008524/train0": 0.04170554131269455, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.7971972227096558, + "307068879/train1": 1.7977794855833054, + "307068879/train2": 1.7964697927236557, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.172493815422058, + "440577711/train0": 1.7942848950624466, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.796036496758461, + "441046872/train1": 1.7939064502716064, + "441046872/train2": 1.7957354187965393, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.835516408085823, + "585904130/train1": 1.843010738492012, + "585904130/train2": 1.706115037202835, + "608016290/train0": 1.3875384330749512, + "608016290/train1": 1.3898992538452148, + "608016290/train2": 1.4294891357421875, + "626486813/train0": 0.1078349556773901, + "626486813/train1": 0.10364562459290028, + "626486813/train2": 0.08551714941859245, + "636680137/train0": 0.06278067082166672, + "636680137/train1": 0.05803162604570389, + "636680137/train2": 0.04585278779268265, + "655884605/train0": 1.842601090669632, + "655884605/train1": 1.8496127128601074, + "655884605/train2": 1.8275748789310455, + "672328262/train0": 1.940206665545702, + "672328262/train1": 1.9466115422546864, + "672328262/train2": 1.9278337731957436, + "701667486/train0": 1.2120786905288696, + "701667486/train1": 1.2052544951438904, + "701667486/train2": 0.0, + "747260437/train0": 0.08635708689689636, + "747260437/train1": 0.144440159201622, + "747260437/train2": 0.09339071810245514, + "750448296/train0": 1.1985591898361843, + "750448296/train1": 1.1875135501225789, + "750448296/train2": 1.1598488887151082, + "767747424/train0": 0.10475751012563705, + "767747424/train1": 0.1008116127923131, + "767747424/train2": 0.09709237236529589 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2779549119689249, + "low": 0.7197053071856498, + "mean": 0.9988301095772872, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9988301095772872, + "per_task": { + "LiftPegUpright-v1": 1.3398529678583144, + "PickCube-v1": 0.9781499902407328, + "PullCube-v1": 0.6881442584097386, + "PushCube-v1": 0.6220440144340197, + "StackCube-v1": 1.365959316943631 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 1.045588894850678, + "low": 0.7773104406013671, + "mean": 0.9091424554741631, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.561831849758682, + "low": 0.561831849758682, + "mean": 0.561831849758682, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.561831849758682, + "per_seed": { + "1073161778/train0": 0.5618318497586814, + "1073161778/train1": 0.5618318497586814, + "1073161778/train2": 0.5618318497586814, + "1113371261/train0": 0.5618318497586814, + "1113371261/train1": 0.5618318497586814, + "1113371261/train2": 0.5618318497586814, + "1151410346/train0": 0.5618318497586814, + "1151410346/train1": 0.5618318497586814, + "1151410346/train2": 0.5618318497586814, + "1261930456/train0": 0.5618318497586814, + "1261930456/train1": 0.5618318497586814, + "1261930456/train2": 0.5618318497586814, + "1324133122/train0": 0.5618318497586814, + "1324133122/train1": 0.5618318497586814, + "1324133122/train2": 0.5618318497586814, + "1324642411/train0": 0.5618318497586814, + "1324642411/train1": 0.5618318497586814, + "1324642411/train2": 0.5618318497586814, + "1375492786/train0": 0.5618318497586814, + "1375492786/train1": 0.5618318497586814, + "1375492786/train2": 0.5618318497586814, + "141984615/train0": 0.5618318497586814, + "141984615/train1": 0.5618318497586814, + "141984615/train2": 0.5618318497586814, + "1461069713/train0": 0.5618318497586814, + "1461069713/train1": 0.5618318497586814, + "1461069713/train2": 0.5618318497586814, + "1480856810/train0": 0.5618318497586814, + "1480856810/train1": 0.5618318497586814, + "1480856810/train2": 0.5618318497586814, + "1509980664/train0": 0.5618318497586814, + "1509980664/train1": 0.5618318497586814, + "1509980664/train2": 0.5618318497586814, + "1522272686/train0": 0.5618318497586814, + "1522272686/train1": 0.5618318497586814, + "1522272686/train2": 0.5618318497586814, + "1549324805/train0": 0.5618318497586814, + "1549324805/train1": 0.5618318497586814, + "1549324805/train2": 0.5618318497586814, + "1578990937/train0": 0.5618318497586814, + "1578990937/train1": 0.5618318497586814, + "1578990937/train2": 0.5618318497586814, + "1747178030/train0": 0.5618318497586814, + "1747178030/train1": 0.5618318497586814, + "1747178030/train2": 0.5618318497586814, + "1790188787/train0": 0.5618318497586814, + "1790188787/train1": 0.5618318497586814, + "1790188787/train2": 0.5618318497586814, + "1822210471/train0": 0.5618318497586814, + "1822210471/train1": 0.5618318497586814, + "1822210471/train2": 0.5618318497586814, + "1845489013/train0": 0.5618318497586814, + "1845489013/train1": 0.5618318497586814, + "1845489013/train2": 0.5618318497586814, + "1999022574/train0": 0.5618318497586814, + "1999022574/train1": 0.5618318497586814, + "1999022574/train2": 0.5618318497586814, + "2012211287/train0": 0.5618318497586814, + "2012211287/train1": 0.5618318497586814, + "2012211287/train2": 0.5618318497586814, + "2038594191/train0": 0.5618318497586814, + "2038594191/train1": 0.5618318497586814, + "2038594191/train2": 0.5618318497586814, + "2054899126/train0": 0.5618318497586814, + "2054899126/train1": 0.5618318497586814, + "2054899126/train2": 0.5618318497586814, + "2061635888/train0": 0.5618318497586814, + "2061635888/train1": 0.5618318497586814, + "2061635888/train2": 0.5618318497586814, + "211245842/train0": 0.5618318497586814, + "211245842/train1": 0.5618318497586814, + "211245842/train2": 0.5618318497586814, + "278008524/train0": 0.5618318497586814, + "278008524/train1": 0.5618318497586814, + "278008524/train2": 0.5618318497586814, + "307068879/train0": 0.5618318497586814, + "307068879/train1": 0.5618318497586814, + "307068879/train2": 0.5618318497586814, + "412102563/train0": 0.5618318497586814, + "412102563/train1": 0.5618318497586814, + "412102563/train2": 0.5618318497586814, + "440577711/train0": 0.5618318497586814, + "440577711/train1": 0.5618318497586814, + "440577711/train2": 0.5618318497586814, + "441046872/train0": 0.5618318497586814, + "441046872/train1": 0.5618318497586814, + "441046872/train2": 0.5618318497586814, + "509430392/train0": 0.5618318497586814, + "509430392/train1": 0.5618318497586814, + "509430392/train2": 0.5618318497586814, + "585904130/train0": 0.5618318497586814, + "585904130/train1": 0.5618318497586814, + "585904130/train2": 0.5618318497586814, + "608016290/train0": 0.5618318497586814, + "608016290/train1": 0.5618318497586814, + "608016290/train2": 0.5618318497586814, + "626486813/train0": 0.5618318497586814, + "626486813/train1": 0.5618318497586814, + "626486813/train2": 0.5618318497586814, + "636680137/train0": 0.5618318497586814, + "636680137/train1": 0.5618318497586814, + "636680137/train2": 0.5618318497586814, + "655884605/train0": 0.5618318497586814, + "655884605/train1": 0.5618318497586814, + "655884605/train2": 0.5618318497586814, + "672328262/train0": 0.5618318497586814, + "672328262/train1": 0.5618318497586814, + "672328262/train2": 0.5618318497586814, + "701667486/train0": 0.5618318497586814, + "701667486/train1": 0.5618318497586814, + "701667486/train2": 0.5618318497586814, + "747260437/train0": 0.5618318497586814, + "747260437/train1": 0.5618318497586814, + "747260437/train2": 0.5618318497586814, + "750448296/train0": 0.5618318497586814, + "750448296/train1": 0.5618318497586814, + "750448296/train2": 0.5618318497586814, + "767747424/train0": 0.5618318497586814, + "767747424/train1": 0.5618318497586814, + "767747424/train2": 0.5618318497586814 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5618318497586817, + "low": 0.5618318497586812, + "mean": 0.5618318497586814, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5618318497586814, + "per_task": { + "LiftPegUpright-v1": 0.5618318497586812, + "PickCube-v1": 0.5618318497586814, + "PullCube-v1": 0.5618318497586821, + "PushCube-v1": 0.5618318497586812, + "StackCube-v1": 0.5618318497586813 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.5618318497586808, + "low": 0.5618318497586808, + "mean": 0.5618318497586808, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "success", + "tau": 0.5618318497586814, + "threshold_scope": "global", + "weights": [ + 0.1080729166666667, + -0.024185830957521734, + 0.048829950545470735, + 0.0024965632066980183, + 0.00835584559796668, + -0.0050463054167126445, + -0.00798844596274325, + -0.05528281312933636, + 0.05962973315558662, + -4.0500568571695844e-16, + 0.006331710613931226, + 0.005480284226139564, + 0.00657288419902417, + -0.01169594171567735, + 0.0025126283456277446, + -0.013446318640521947, + -0.008947166640087222, + 0.004239781773056478, + 0.006331710613932057, + 0.005480284226140768, + 0.006572884199024171, + -0.011695941715677966, + 0.0025126283456291636, + -0.01344631864052295, + -0.00894716664008866, + 0.004239781773057182, + 0.0020995116128919127, + 0.0002466889367765306, + 0.000564873352711312, + -0.007179935281171779, + -0.0023989698170510925, + -0.005260066021344833, + -0.011224893763853489, + 0.0029797881312846053, + 1.8821359254733254e-17, + -0.00460493044514983, + -0.011111772840711583, + -0.13507425466995532, + 0.01057942922751523, + -0.13281704782800485, + -0.05179099543916294, + -0.014289442083854131, + -0.0026811672398937975, + -0.010174377719871314, + 0.03832773345245116, + 0.04772143068889488, + -0.009141272841762055, + -0.03984228092769289, + 0.04401106088504414, + 0.10578161010564757, + -0.03325205538738006, + -0.0584534622989398, + -0.016878116190133963, + -0.027430647707732236, + -0.0026018961589524964, + 0.012649592465912941, + -0.06212201438733678, + -0.05147153041515409, + -0.007861849255584086, + 0.015919637915346244, + -0.006405784950480057, + -0.024915106901981066, + 0.011980114296912925, + -0.03826578385385104, + 0.023749892485422253, + 0.07237484095245582, + -0.035574252275188935, + -0.02773336036862209, + -0.02198577130819901, + -0.04293178706427603, + 0.016645793514863727, + 0.09660939671125379, + -0.014040559067523162, + -0.016172185196134906, + 0.04413887342508846, + 0.016086265191531834, + 0.006607525106538642, + 0.027448878654304787, + -0.00818014949451984 + ] +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..ebf5b3e638502ec85cf4e5f766ca96312531c780 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics_by_seed.json @@ -0,0 +1,3234 @@ +{ + "1073161778/train0": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5092992512408128, + "pairwise_causal_calibration_ece": 0.15215639409795562, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.20584099091859903, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7180102467536926, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2819897532463074, + "tau": 0.5618318497586814 + }, + "1073161778/train1": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.506518118161019, + "pairwise_causal_calibration_ece": 0.11366097530387614, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19554991093790747, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7134289145469666, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2865710854530334, + "tau": 0.5618318497586814 + }, + "1073161778/train2": { + "base_success": 0.0, + "base_utility": 0.725517988204956, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5082470514492481, + "pairwise_causal_calibration_ece": 0.2939613371635338, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19441156679128385, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725517988204956, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274482011795044, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5120817381301231, + "pairwise_causal_calibration_ece": 0.04779602384440884, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11841214667429321, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0740400105714798, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9259599894285202, + "tau": 0.5618318497586814 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5114294633513058, + "pairwise_causal_calibration_ece": 0.1671419652201228, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.111487318938856, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09622243046760559, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9037775695323944, + "tau": 0.5618318497586814 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.11109928786754608, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5208647930155366, + "pairwise_causal_calibration_ece": 0.05056377841303483, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1637967593188533, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1003647893667221, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11109928786754608, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.899635210633278, + "tau": 0.5618318497586814 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5084242760212154, + "pairwise_causal_calibration_ece": 0.20586143826449887, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14682773551354072, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5684944987297058, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3400209844112396, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4315055012702942, + "tau": 0.5618318497586814 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5116087729702029, + "pairwise_causal_calibration_ece": 0.047323058684488606, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1505309920780672, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2439272701740265, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.015453755855560303, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7560727298259735, + "tau": 0.5618318497586814 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2284735143184662, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5093949827060259, + "pairwise_causal_calibration_ece": 0.0977478744368312, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15717334321041485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5630103945732117, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2284735143184662, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3345368802547455, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4369896054267883, + "tau": 0.5618318497586814 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.507207152590294, + "pairwise_causal_calibration_ece": 0.00720715259029403, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.006272514778440541, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09678153693675995, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.90321846306324, + "tau": 0.5618318497586814 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.509870803335365, + "pairwise_causal_calibration_ece": 0.025843482378920646, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.077102113380139, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997129440307617, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9000287055969238, + "tau": 0.5618318497586814 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5205192822349104, + "pairwise_causal_calibration_ece": 0.12766213937776755, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09402188754699127, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08545441925525665, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9145455807447433, + "tau": 0.5618318497586814 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5156896062548352, + "pairwise_causal_calibration_ece": 0.05573896517373622, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07612377904669673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0628519058227539, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.937148094177246, + "tau": 0.5618318497586814 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5077543727318965, + "pairwise_causal_calibration_ece": 0.2422456272681035, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21872963666248138, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0843733698129654, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.00014631450176239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9156266301870346, + "tau": 0.5618318497586814 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.084227055311203, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5106739762139129, + "pairwise_causal_calibration_ece": 0.08210254764248431, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0939715257270842, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09815146028995514, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.084227055311203, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013924404978752136, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9018485397100449, + "tau": 0.5618318497586814 + }, + "1324642411/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5213088389133544, + "pairwise_causal_calibration_ece": 0.1284516960562116, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.8928362396780647, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21000581979751587, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14966940879821777, + "selected_utility_gain_over_base": -1.8503305912017822, + "selector_regret": 0.060336410999298096, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7899941802024841, + "tau": 0.5618318497586814 + }, + "1324642411/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.18518518518518517, + "pairwise_causal_calibration_confidence": 0.5212524000863837, + "pairwise_causal_calibration_ece": 0.33606721490119856, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.7531487720723857, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.17775490880012512, + "selected_utility_gain_over_base": -1.8222450911998749, + "selector_regret": 1.8222450911998749, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1324642411/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.16666666666666666, + "pairwise_causal_calibration_confidence": 0.527095259436081, + "pairwise_causal_calibration_ece": 0.3604285927694144, + "pairwise_causal_calibration_pairs": 18.0, + "predicted_margin": 0.6491618422591205, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14906781911849976, + "selected_utility_gain_over_base": -1.8509321808815002, + "selector_regret": 1.8509321808815002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6923076923076923, + "pairwise_causal_calibration_confidence": 0.5390208488024293, + "pairwise_causal_calibration_ece": 0.153286843505263, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.23249138223608884, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5136001990069363, + "pairwise_causal_calibration_ece": 0.3078283724216351, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.037632419519960006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14743654429912567, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8525634557008743, + "tau": 0.5618318497586814 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5233704587833103, + "pairwise_causal_calibration_ece": 0.15520096978811826, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17970484878502413, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19452734291553497, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.805472657084465, + "tau": 0.5618318497586814 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5150349488116895, + "pairwise_causal_calibration_ece": 0.1578920916688324, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18927476177277072, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.023719489574432373, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2620577812194824, + "tau": 0.5618318497586814 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5209787160422698, + "pairwise_causal_calibration_ece": 0.05669300175655545, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18318239803192132, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04397805407643318, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2417992167174816, + "tau": 0.5618318497586814 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.16783273220062256, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5266278793488347, + "pairwise_causal_calibration_ece": 0.38377073649169186, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.114212135741804, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04447716474533081, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16783273220062256, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24130010604858398, + "tau": 0.5618318497586814 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5120775991497745, + "pairwise_causal_calibration_ece": 0.05935097227879693, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.21845740942913472, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05616961047053337, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10317284241318703, + "tau": 0.5618318497586814 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5159485485371296, + "pairwise_causal_calibration_ece": 0.05548002289144183, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.30554439462118227, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05644957348704338, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10289287939667702, + "tau": 0.5618318497586814 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.22852759063243866, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5323058563946664, + "pairwise_causal_calibration_ece": 0.07483700074819066, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5338556706581634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11542954295873642, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22852759063243866, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04391290992498398, + "tau": 0.5618318497586814 + }, + "1480856810/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5131988287429138, + "pairwise_causal_calibration_ece": 0.15605597160005663, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7158970211825535, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14178013801574707, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12667706608772278, + "selected_utility_gain_over_base": -1.8733229339122772, + "selector_regret": 0.015103071928024292, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06770022213459015, + "tau": 0.5618318497586814 + }, + "1480856810/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5316435268709924, + "pairwise_causal_calibration_ece": 0.13878638401384952, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.7444206849830362, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.184015691280365, + "selected_utility_gain_over_base": -1.815984308719635, + "selector_regret": 1.815984308719635, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1480856810/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5203685487667845, + "pairwise_causal_calibration_ece": 0.08677430837607258, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5127510371288293, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19622710347175598, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.013253256678581238, + "tau": 0.5618318497586814 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.5069861149914309, + "pairwise_causal_calibration_ece": 0.13301388500856914, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.1645139076256539, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5077947588157862, + "pairwise_causal_calibration_ece": 0.10779475881578615, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.12250900335942658, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.167920172214508, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8320798277854919, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5080683702895407, + "pairwise_causal_calibration_ece": 0.06336020113903074, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1805548023297982, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.83889240026474, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8320798277854919, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006812572479248047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.16110759973526, + "tau": 0.5618318497586814 + }, + "1522272686/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5308364411582983, + "pairwise_causal_calibration_ece": 0.38797929830115546, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4948526502859529, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5233199596405029, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.476680040359497, + "tau": 0.5618318497586814 + }, + "1522272686/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5263848235942014, + "pairwise_causal_calibration_ece": 0.24067053787991566, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3317013229350659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5422300696372986, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4577699303627014, + "tau": 0.5618318497586814 + }, + "1522272686/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5271177549323166, + "pairwise_causal_calibration_ece": 0.1157393879248263, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3719790973396491, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5465855152184887, + "pairwise_causal_calibration_ece": 0.20341448478151117, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4099932021305159, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21791090071201324, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04541558027267456, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07954518496990204, + "tau": 0.5618318497586814 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5489097792855695, + "pairwise_causal_calibration_ece": 0.3082330778572876, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.4366493356305453, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.3088321089744568, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1363367885351181, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.17249532043933868, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5156532908328474, + "pairwise_causal_calibration_ece": 0.0914895663100097, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.33492807900387084, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21403594315052032, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17249532043933868, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04154062271118164, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08342014253139496, + "tau": 0.5618318497586814 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5120526217939588, + "pairwise_causal_calibration_ece": 0.05937594963461257, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12075832280916365, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21425709128379822, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857429087162018, + "tau": 0.5618318497586814 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.517049827852834, + "pairwise_causal_calibration_ece": 0.37580731500430886, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15613350902346218, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21763037145137787, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0033643990755081177, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7823696285486221, + "tau": 0.5618318497586814 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.21426597237586975, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5121887042635905, + "pairwise_causal_calibration_ece": 0.09495415287926656, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.14872603579685487, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23259200155735016, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21426597237586975, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018326029181480408, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7674079984426498, + "tau": 0.5618318497586814 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.6873128160326993, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.9835802031786316, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.9536859431190323, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1790188787/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5223174936773409, + "pairwise_causal_calibration_ece": 0.15625393489408768, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.25334075772729825, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1790188787/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5184372952384458, + "pairwise_causal_calibration_ece": 0.23156270476155416, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.27780618403492885, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1862330436706543, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8137669563293457, + "tau": 0.5618318497586814 + }, + "1790188787/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.5109241069247938, + "pairwise_causal_calibration_ece": 0.17089407489338793, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.18703346552659444, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5138796533820333, + "pairwise_causal_calibration_ece": 0.057548918046538056, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0577847490536553, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03802649304270744, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9619735069572926, + "tau": 0.5618318497586814 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5083982289146939, + "pairwise_causal_calibration_ece": 0.07982680034326534, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.07212192716463756, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07539409399032593, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.924605906009674, + "tau": 0.5618318497586814 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.09442546218633652, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5174993924353067, + "pairwise_causal_calibration_ece": 0.26749939243530674, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0031507691650567438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058819372206926346, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09442546218633652, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9411806277930737, + "tau": 0.5618318497586814 + }, + "1845489013/train0": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5068891153127965, + "pairwise_causal_calibration_ece": 0.17168231325863215, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.05982643176183276, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0428844690322876, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9571155309677124, + "tau": 0.5618318497586814 + }, + "1845489013/train1": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5120112216374724, + "pairwise_causal_calibration_ece": 0.08343979306604382, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0787154239384655, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09529292583465576, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9047070741653442, + "tau": 0.5618318497586814 + }, + "1845489013/train2": { + "base_success": 0.0, + "base_utility": 0.11255800724029541, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5049839841564138, + "pairwise_causal_calibration_ece": 0.29069826987069947, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.003957935292008923, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0861951932311058, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11255800724029541, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9138048067688942, + "tau": 0.5618318497586814 + }, + "1999022574/train0": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5125355587394795, + "pairwise_causal_calibration_ece": 0.20175015554623477, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19337117648183325, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5970457196235657, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4029542803764343, + "tau": 0.5618318497586814 + }, + "1999022574/train1": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5081622486924189, + "pairwise_causal_calibration_ece": 0.07959082012099034, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.09814524693113953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1720311045646667, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "1999022574/train2": { + "base_success": 0.0, + "base_utility": 0.8279688954353333, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5126369132288152, + "pairwise_causal_calibration_ece": 0.20164880105689909, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1565539695903909, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7970014810562134, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8279688954353333, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2029985189437866, + "tau": 0.5618318497586814 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5090347138321765, + "pairwise_causal_calibration_ece": 0.044748999546462165, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18515916980019373, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7277753949165344, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2722246050834656, + "tau": 0.5618318497586814 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.5141496618096716, + "pairwise_causal_calibration_ece": 0.2919274395874494, + "pairwise_causal_calibration_pairs": 27.0, + "predicted_margin": 0.13504095940734823, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2717533111572266, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7282466888427734, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5108011563830763, + "pairwise_causal_calibration_ece": 0.18937258495450487, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12797764508486972, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7270426154136658, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7282466888427734, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2729573845863342, + "tau": 0.5618318497586814 + }, + "2038594191/train0": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5291333104927968, + "pairwise_causal_calibration_ece": 0.13627616763565392, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07343156967546631, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15247176587581635, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0574311688542366, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8475282341241837, + "tau": 0.5618318497586814 + }, + "2038594191/train1": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5272185314816756, + "pairwise_causal_calibration_ece": 0.11563861137546727, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.11157849082379764, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1902807205915451, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09524012356996536, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.809719279408455, + "tau": 0.5618318497586814 + }, + "2038594191/train2": { + "base_success": 0.0, + "base_utility": 0.09504059702157974, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5202926621233347, + "pairwise_causal_calibration_ece": 0.05600694783762039, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.04359967653884958, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.24880830943584442, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09504059702157974, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.15376771241426468, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7511916905641556, + "tau": 0.5618318497586814 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5096744733127383, + "pairwise_causal_calibration_ece": 0.03087704905791655, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.38208593853249617, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0726270377635956, + "selected_candidate_index": 0.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9273729622364044, + "tau": 0.5618318497586814 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5220705265713083, + "pairwise_causal_calibration_ece": 0.0646066918570827, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.5828384224098009, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0755530372262, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0613084435462952, + "selected_utility_gain_over_base": -0.9386915564537048, + "selector_regret": 0.014244593679904938, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9244469627737999, + "tau": 0.5618318497586814 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5259225813730811, + "pairwise_causal_calibration_ece": 0.11203870931345944, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.4502950976842904, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0683744624257088, + "selected_candidate_index": 1.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0612773001194, + "selected_utility_gain_over_base": -0.9387226998806, + "selector_regret": 0.007097162306308746, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9316255375742912, + "tau": 0.5618318497586814 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5035728957399334, + "pairwise_causal_calibration_ece": 0.13928424711720955, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.08547837727895316, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07642464339733124, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9235753566026688, + "tau": 0.5618318497586814 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5095019143713128, + "pairwise_causal_calibration_ece": 0.27621237134297294, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03930678750007065, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09046469628810883, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.004257768392562866, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9095353037118912, + "tau": 0.5618318497586814 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.08620692789554596, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5079491341690449, + "pairwise_causal_calibration_ece": 0.24205086583095514, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.008004389977088786, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07902413606643677, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08620692789554596, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9209758639335632, + "tau": 0.5618318497586814 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5129474725633024, + "pairwise_causal_calibration_ece": 0.09419538457955468, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22532734972882426, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5822281837463379, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.3979237377643585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.417771816253662, + "tau": 0.5618318497586814 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5130796019472789, + "pairwise_causal_calibration_ece": 0.12977754090986404, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22424307526810303, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2016763538122177, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.017371907830238342, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7983236461877823, + "tau": 0.5618318497586814 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.18430444598197937, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5205888094479262, + "pairwise_causal_calibration_ece": 0.12226833340921672, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2206154553304457, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2142850160598755, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18430444598197937, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.029980570077896118, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7857149839401245, + "tau": 0.5618318497586814 + }, + "278008524/train0": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5223926438814469, + "pairwise_causal_calibration_ece": 0.011196321940723453, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.215222220593497, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0722030997276306, + "selected_candidate_index": 4.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04170554131269455, + "tau": 0.5618318497586814 + }, + "278008524/train1": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5243015603781166, + "pairwise_causal_calibration_ece": 0.1128492198109417, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.1986640133517516, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1246033310890198, + "selected_candidate_index": 1.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "278008524/train2": { + "base_success": 0.5, + "base_utility": 1.1250072121620178, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.514035350060809, + "pairwise_causal_calibration_ece": 0.11798232496959549, + "pairwise_causal_calibration_pairs": 14.0, + "predicted_margin": 0.252213252965857, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.157032236456871, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.1250072121620178, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03202502429485321, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5119848106472845, + "pairwise_causal_calibration_ece": 0.19055623921871306, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.005683666382895997, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20280277729034424, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7971972227096558, + "tau": 0.5618318497586814 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5075913402530835, + "pairwise_causal_calibration_ece": 0.0433056259673692, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.01192475167673359, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20222051441669464, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7977794855833054, + "tau": 0.5618318497586814 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.20915235579013824, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.50920166710305, + "pairwise_causal_calibration_ece": 0.09794119003980706, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.003886783551851819, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2035302072763443, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20915235579013824, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7964697927236557, + "tau": 0.5618318497586814 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5149085793100113, + "pairwise_causal_calibration_ece": 0.08633715073858278, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.22150303258309284, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5120484995354441, + "pairwise_causal_calibration_ece": 0.04776278524972977, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12051079262473083, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5130901931270576, + "pairwise_causal_calibration_ece": 0.08451876455562907, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15822655084813653, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8275061845779419, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.172493815422058, + "tau": 0.5618318497586814 + }, + "440577711/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5163938105237615, + "pairwise_causal_calibration_ece": 0.12646333233338136, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.5586790894555644, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2057151049375534, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7942848950624466, + "tau": 0.5618318497586814 + }, + "440577711/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5239216972771956, + "pairwise_causal_calibration_ece": 0.5239216972771956, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": 0.660329471290032, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.21395862102508545, + "selected_utility_gain_over_base": -1.7860413789749146, + "selector_regret": 1.7860413789749146, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "440577711/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.6996994006285734, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.510976779082865, + "pairwise_causal_calibration_ece": 0.024737506631420714, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.006895278681057632, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.203963503241539, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.796036496758461, + "tau": 0.5618318497586814 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5088684124078007, + "pairwise_causal_calibration_ece": 0.06256015902077072, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.03548490283103705, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20609354972839355, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7939064502716064, + "tau": 0.5618318497586814 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.2097676396369934, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5127281442153561, + "pairwise_causal_calibration_ece": 0.16584328435607254, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.0013491370718414428, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2042645812034607, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2097676396369934, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7957354187965393, + "tau": 0.5618318497586814 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.28, + "pairwise_causal_calibration_confidence": 0.5167159497943329, + "pairwise_causal_calibration_ece": 0.23671594979433286, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": 0.40670225613190863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.09090909090909091, + "pairwise_causal_calibration_confidence": 0.5152174445598239, + "pairwise_causal_calibration_ece": 0.42430835365073294, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.46272796115382003, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 0.04073460400104523, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.09090909090909091, + "pairwise_causal_calibration_confidence": 0.513267600062783, + "pairwise_causal_calibration_ece": 0.4223585091536921, + "pairwise_causal_calibration_pairs": 22.0, + "predicted_margin": 0.4020750348567389, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04073460400104523, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9592653959989548, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5316537240199258, + "pairwise_causal_calibration_ece": 0.14691770455150277, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15837838927275857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.16448359191417694, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09344705194234848, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.835516408085823, + "tau": 0.5618318497586814 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5109791799445677, + "pairwise_causal_calibration_ece": 0.1538363228017105, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07464422139595693, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15698926150798798, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08595272153615952, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.843010738492012, + "tau": 0.5618318497586814 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.07103653997182846, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5257097365720064, + "pairwise_causal_calibration_ece": 0.06142402228629207, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12634263058484524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2938849627971649, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07103653997182846, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.22284842282533646, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.706115037202835, + "tau": 0.5618318497586814 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5141263992957481, + "pairwise_causal_calibration_ece": 0.12126925643860526, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2141408309665893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6124615669250488, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08710145950317383, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3875384330749512, + "tau": 0.5618318497586814 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5114864515125813, + "pairwise_causal_calibration_ece": 0.36862930865543847, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19953791912822288, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6101007461547852, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.08474063873291016, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3898992538452148, + "tau": 0.5618318497586814 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.525360107421875, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5207029573796137, + "pairwise_causal_calibration_ece": 0.41356010023675654, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.24661642979812717, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5705108642578125, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.525360107421875, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0451507568359375, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4294891357421875, + "tau": 0.5618318497586814 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48214285714285715, + "pairwise_causal_calibration_confidence": 0.5206261667386757, + "pairwise_causal_calibration_ece": 0.26398015577572026, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2289130081988947, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05774742551147938, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04174089431762695, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.016006531193852425, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1078349556773901, + "tau": 0.5618318497586814 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.517265264607039, + "pairwise_causal_calibration_ece": 0.0748773088696959, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2919681543477006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0619367565959692, + "selected_candidate_index": 1.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04174089431762695, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.020195862278342247, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10364562459290028, + "tau": 0.5618318497586814 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.04174089431762695, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5332777485074317, + "pairwise_causal_calibration_ece": 0.2832777485074317, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.29076199506255523, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08006523177027702, + "selected_candidate_index": 2.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04174089431762695, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03832433745265007, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08551714941859245, + "tau": 0.5618318497586814 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5172753273454199, + "pairwise_causal_calibration_ece": 0.12441818448827707, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.19058677054848244, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10778365284204483, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.06278067082166672, + "tau": 0.5618318497586814 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.521068070904193, + "pairwise_causal_calibration_ece": 0.1282109280470502, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12268453237476608, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11253269761800766, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05803162604570389, + "tau": 0.5618318497586814 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.11957359313964844, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5248912478073117, + "pairwise_causal_calibration_ece": 0.20346267637874021, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3268104640308388, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1247115358710289, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11957359313964844, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005137942731380463, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04585278779268265, + "tau": 0.5618318497586814 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5125805300949516, + "pairwise_causal_calibration_ece": 0.16599089847647697, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03266312735678528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15739890933036804, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.01332351565361023, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.842601090669632, + "tau": 0.5618318497586814 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.506450790055417, + "pairwise_causal_calibration_ece": 0.10069206708744005, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.03338313576240709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15038728713989258, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006311893463134766, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8496127128601074, + "tau": 0.5618318497586814 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.1440753936767578, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.505842678549458, + "pairwise_causal_calibration_ece": 0.29155696426374367, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.07788535796650367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17242512106895447, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1440753936767578, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.028349727392196655, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8275748789310455, + "tau": 0.5618318497586814 + }, + "672328262/train0": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5165144952070263, + "pairwise_causal_calibration_ece": 0.2334855047929737, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.02720919346098577, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05979333445429802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.940206665545702, + "tau": 0.5618318497586814 + }, + "672328262/train1": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.07142857142857142, + "pairwise_causal_calibration_confidence": 0.5183234734272898, + "pairwise_causal_calibration_ece": 0.4468949019987184, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.047096220474437275, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.053388457745313644, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9466115422546864, + "tau": 0.5618318497586814 + }, + "672328262/train2": { + "base_success": 0.0, + "base_utility": 0.10593423992395401, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5090400478771011, + "pairwise_causal_calibration_ece": 0.02667423783718459, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0702178669924084, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07216622680425644, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10593423992395401, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9278337731957436, + "tau": 0.5618318497586814 + }, + "701667486/train0": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5106631871119919, + "pairwise_causal_calibration_ece": 0.06076538431657952, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18005163348657682, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7879213094711304, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2120786905288696, + "tau": 0.5618318497586814 + }, + "701667486/train1": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5098775552568305, + "pairwise_causal_calibration_ece": 0.02583673045745516, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.12475135370966416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7947455048561096, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0034714341163635254, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2052544951438904, + "tau": 0.5618318497586814 + }, + "701667486/train2": { + "base_success": 0.0, + "base_utility": 0.7912740707397461, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5194690186587149, + "pairwise_causal_calibration_ece": 0.08767383848414223, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2642366239473226, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7912740707397461, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.208725929260254, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.5618318497586814 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5120181933719066, + "pairwise_causal_calibration_ece": 0.11916105051476372, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1795266808160827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19230616092681885, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08635708689689636, + "tau": 0.5618318497586814 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5088316663078858, + "pairwise_causal_calibration_ece": 0.02688261940639991, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.17198143779324082, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1342230886220932, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.144440159201622, + "tau": 0.5618318497586814 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5073520007934037, + "pairwise_causal_calibration_ece": 0.3497908563494534, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.1459627586815193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18527252972126007, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09339071810245514, + "tau": 0.5618318497586814 + }, + "750448296/train0": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5233276622092005, + "pairwise_causal_calibration_ece": 0.104589703577334, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3509091319089219, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20943730572859445, + "selected_candidate_index": 2.3333333333333335, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06344164411226909, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1985591898361843, + "tau": 0.5618318497586814 + }, + "750448296/train1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5833333333333334, + "pairwise_causal_calibration_confidence": 0.5305656417959553, + "pairwise_causal_calibration_ece": 0.09714266836899327, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3670406155313691, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22532296180725098, + "selected_candidate_index": 2.3333333333333335, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06390197078386943, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1875135501225789, + "tau": 0.5618318497586814 + }, + "750448296/train2": { + "base_success": 0.3333333333333333, + "base_utility": 0.7605033268531164, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.5833333333333334, + "pairwise_causal_calibration_confidence": 0.526807577707075, + "pairwise_causal_calibration_ece": 0.05652575562625828, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.35168270720155403, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2481476068496704, + "selected_candidate_index": 3.0, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7605033268531164, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.11175449689229329, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 1.1598488887151082, + "tau": 0.5618318497586814 + }, + "767747424/train0": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.45535714285714285, + "pairwise_causal_calibration_confidence": 0.524268127667919, + "pairwise_causal_calibration_ece": 0.1905210480971844, + "pairwise_causal_calibration_pairs": 22.75, + "predicted_margin": 0.3467289216925133, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5789013803005219, + "selected_candidate_index": 4.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0861943028867245, + "selected_utility_gain_over_base": 0.011485546827316284, + "selector_regret": 0.007970128208398819, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10475751012563705, + "tau": 0.5618318497586814 + }, + "767747424/train1": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.44642857142857145, + "pairwise_causal_calibration_confidence": 0.5309237590932049, + "pairwise_causal_calibration_ece": 0.12149106073317761, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.269433473428486, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5825987188145518, + "selected_candidate_index": 2.75, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0747087560594082, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019207116216421127, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1008116127923131, + "tau": 0.5618318497586814 + }, + "767747424/train2": { + "base_success": 0.5, + "base_utility": 1.0747087560594082, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.4017857142857143, + "pairwise_causal_calibration_confidence": 0.5186143981913337, + "pairwise_causal_calibration_ece": 0.14789818559258952, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.2226382768839351, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5809952197596431, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.0747087560594082, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013884376734495163, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09709237236529589, + "tau": 0.5618318497586814 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..e19f92d94309785d3d5bf3190410d9863e966b75 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.0, + "base_utility": 0.4796089559793472, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.3333333333333333, + "pairwise_causal_calibration_accuracy": 0.3862433862433862, + "pairwise_causal_calibration_confidence": 0.5110477943182259, + "pairwise_causal_calibration_ece": 0.17161536008124903, + "pairwise_causal_calibration_pairs": 27.933333333333334, + "predicted_margin": 0.11036931560956814, + "proposal_oracle_success": 0.13333333333333333, + "proposal_oracle_utility": 0.6601470321416855, + "selected_candidate_index": 4.733333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.4796089559793472, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.18421521186828613, + "success_selector_gap": 0.13333333333333333, + "success_support_gap": 0.8666666666666667, + "support_gap": 1.3398529678583144, + "tau": 0.5618318497586812 + }, + "PickCube-v1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5281130065616054, + "pairwise_causal_calibration_ece": 0.24812974136863244, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.3995110235202226, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 1.021850009759267, + "selected_candidate_index": 1.3333333333333333, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.6666666666666666, + "support_gap": 0.9781499902407328, + "tau": 0.5618318497586814 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.8032506120204925, + "coverage": 0.08, + "execute_generated": 0.08, + "fallback_rate": 0.92, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.32, + "pairwise_causal_calibration_accuracy": 0.5088661338661338, + "pairwise_causal_calibration_confidence": 0.522002123999376, + "pairwise_causal_calibration_ece": 0.13393386645915098, + "pairwise_causal_calibration_pairs": 24.16, + "predicted_margin": 0.26581590627488516, + "proposal_oracle_success": 0.17333333333333334, + "proposal_oracle_utility": 0.46170220340291657, + "selected_candidate_index": 2.88, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": -0.02666666666666667, + "selected_utility": 0.7537987943490346, + "selected_utility_gain_over_base": -0.049451817671457926, + "selector_regret": 0.027659279704093934, + "success_selector_gap": 0.0, + "success_support_gap": 0.3466666666666667, + "support_gap": 0.6881442584097386, + "tau": 0.5618318497586821 + }, + "PushCube-v1": { + "base_success": 0.8333333333333334, + "base_utility": 1.6734557673335075, + "coverage": 0.3888888888888889, + "execute_generated": 0.3888888888888889, + "fallback_rate": 0.6111111111111112, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.16666666666666666, + "pairwise_causal_calibration_accuracy": 0.3967934244404833, + "pairwise_causal_calibration_confidence": 0.5182152964138562, + "pairwise_causal_calibration_ece": 0.20331974993456392, + "pairwise_causal_calibration_pairs": 23.833333333333332, + "predicted_margin": 0.46411142965639, + "proposal_oracle_success": 0.5555555555555556, + "proposal_oracle_utility": 1.1790093589160178, + "selected_candidate_index": 4.166666666666667, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.3333333333333333, + "selected_utility": 1.0624081848396196, + "selected_utility_gain_over_base": -0.611047582493888, + "selector_regret": 0.7349132572611173, + "success_selector_gap": 0.3888888888888889, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6220440144340197, + "tau": 0.5618318497586812 + }, + "StackCube-v1": { + "base_success": 0.09090909090909091, + "base_utility": 0.5003613992170854, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6060606060606061, + "pairwise_causal_calibration_accuracy": 0.5877922077922076, + "pairwise_causal_calibration_confidence": 0.5114535745455024, + "pairwise_causal_calibration_ece": 0.12935577712039273, + "pairwise_causal_calibration_pairs": 27.818181818181817, + "predicted_margin": 0.13404695454329726, + "proposal_oracle_success": 0.18181818181818182, + "proposal_oracle_utility": 0.634040683056369, + "selected_candidate_index": 4.151515151515151, + "selected_success": 0.09090909090909091, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5003613992170854, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.18030828102068466, + "success_selector_gap": 0.12121212121212122, + "success_support_gap": 0.8181818181818182, + "support_gap": 1.365959316943631, + "tau": 0.5618318497586813 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..750fe062d7a14cdda4749e110a4ab3261c146499 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.0903 & 0.9097 & 0.3333 & 0.2778 & 0.2222 & 0.3611 & 0.5139 & 0.0903 & 0.0152 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b8b75d654cfd32307e35e44ca29c24573f356646 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_perdim_trainmax_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=0.0 +tau=0.561832 +calibration_input=runs/ctt_base_context_obs_train_cal_perdim_trainmax_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/command.txt index 7ae83b5ae0dad5a19d64a2fe5e0e56dd8bc4fb45..87f7be50fcedff73f6f142144abe56df3ccb4eb6 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/command.txt +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/command.txt @@ -1 +1 @@ -python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test --feature-set context_tangent --target success --k 8 --bootstrap-samples 1000 +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test --feature-set context_tangent --target success --k 8 --bootstrap-samples 1000 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/config.yaml index ea78a1d96f9bcb427ee12131afd8a4fad9c8c88a..73f733c7ef5c6728049d33e1045fed6ac1d23ee4 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/config.yaml +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/config.yaml @@ -5,7 +5,14 @@ checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt eval_input: runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json eval_target_index: data/cil_charts_rgb_refs/test/index.json feature_set: context_tangent +fit_objective: pointwise k: 8 +no_markdown_report: True out_dir: runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test +pairwise_weight: 1.0 ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: None +success_bonus: 1.0 target: success +threshold_scope: global diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics.json index cf868a88d05c98e39755e74d155ed2c973c6bb96..45bb90857f76a42b6ddd23ad827f0b7af72dc31b 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics.json @@ -26,6 +26,10 @@ "hidden_chart_oracle_success": 0.6597222222222222, "hidden_chart_oracle_utility": 1.4150809271054134, "outcome_ptr": 0.3773148148148148, + "pairwise_causal_calibration_accuracy": 0.5263051396195872, + "pairwise_causal_calibration_confidence": 0.5155242157559488, + "pairwise_causal_calibration_ece": 0.010950533380881767, + "pairwise_causal_calibration_pairs": 9884.0, "proposal_oracle_success": 0.36342592592592593, "proposal_oracle_utility": 0.8099956938270751, "selected_success": 0.38657407407407407, @@ -49,6 +53,10 @@ "hidden_chart_oracle_success": 0.7291666666666666, "hidden_chart_oracle_utility": 1.5183161981403828, "outcome_ptr": 0.4166666666666667, + "pairwise_causal_calibration_accuracy": 0.47594202898550725, + "pairwise_causal_calibration_confidence": 0.5151423766525927, + "pairwise_causal_calibration_ece": 0.03920034766708541, + "pairwise_causal_calibration_pairs": 3450.0, "proposal_oracle_success": 0.3819444444444444, "proposal_oracle_utility": 0.8643959594224321, "selected_success": 0.3402777777777778, @@ -305,11 +313,203 @@ 0.8193685011254338, 0.784874822648645 ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 3456, + "num_fit_rows": 3456, + "num_pairwise_rows": 0, + "num_pointwise_rows": 3456, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", "k": 8, "num_calibration_candidates": 3456, "num_calibration_rows": 432, "num_eval_candidates": 1152, "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5263051396195872, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.010869027598666037, + "accuracy": 0.5263637283675742, + "confidence": 0.5154947007689081, + "count": 9881, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.27940341140542274, + "accuracy": 0.3333333333333333, + "confidence": 0.6127367447387561, + "count": 3, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.010950533380881767, + "mean_confidence": 0.5155242157559488, + "n_bins": 10, + "num_pairs": 9884, + "num_rows": 432 + }, + "eval": { + "accuracy": 0.47594202898550725, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.03920034766708541, + "accuracy": 0.47594202898550725, + "confidence": 0.5151423766525927, + "count": 3450, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.03920034766708541, + "mean_confidence": 0.5151423766525927, + "n_bins": 10, + "num_pairs": 3450, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, "report_type": "learned_dominance_selector_eval", "ridge_lambdas": [ 0.0, @@ -330,6 +530,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5099995097795701, + "pairwise_causal_calibration_ece": 0.061429061649001304, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15525138726863855, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -357,6 +561,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5154469606567962, + "pairwise_causal_calibration_ece": 0.015446960656796183, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12488897315305114, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20391923189163208, @@ -384,6 +592,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.507958960013907, + "pairwise_causal_calibration_ece": 0.25795896001390695, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17450214530695946, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -411,6 +623,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5080911020548196, + "pairwise_causal_calibration_ece": 0.31333746937375184, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19628991784756858, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -438,6 +654,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5157279739172514, + "pairwise_causal_calibration_ece": 0.26572797391725145, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1335357706256974, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5837622880935669, @@ -465,6 +685,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5247878254029282, + "pairwise_causal_calibration_ece": 0.34621639683149963, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14162108969262518, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16821590065956116, @@ -492,6 +716,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.7523816581368387, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -519,6 +747,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.21099537685281347, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -546,6 +778,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5163492540805769, + "pairwise_causal_calibration_ece": 0.12349211122343401, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12959505268620136, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06010149046778679, @@ -573,6 +809,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5154101559097269, + "pairwise_causal_calibration_ece": 0.12255301305258409, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2644640154439865, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.012636144645512104, @@ -600,6 +840,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5254447839705787, + "pairwise_causal_calibration_ece": 0.0816980731722784, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6665486720419214, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15609906613826752, @@ -627,6 +871,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6363636363636364, + "pairwise_causal_calibration_confidence": 0.5121235763357475, + "pairwise_causal_calibration_ece": 0.1242400600278889, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.037000828296731625, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -654,6 +902,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5197389648498357, + "pairwise_causal_calibration_ece": 0.08740389229302137, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.705199845567416, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16806542873382568, @@ -681,6 +933,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5151801795860341, + "pairwise_causal_calibration_ece": 0.015180179586034126, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15942683756676435, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07040411233901978, @@ -708,6 +964,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5077057701570821, + "pairwise_causal_calibration_ece": 0.00770577015708207, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13991927423910702, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02055191993713379, @@ -735,6 +995,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5146880096926524, + "pairwise_causal_calibration_ece": 0.3004022954069381, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21453128771326116, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07324296236038208, @@ -762,6 +1026,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5086091100493885, + "pairwise_causal_calibration_ece": 0.06281946137918293, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14127758503794915, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -789,6 +1057,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7222222222222222, + "pairwise_causal_calibration_confidence": 0.5232513345220312, + "pairwise_causal_calibration_ece": 0.198970887700191, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.16777309899666812, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -816,6 +1088,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5232574572379338, + "pairwise_causal_calibration_ece": 0.23754317152364812, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20868979202742993, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.008802552707493305, @@ -843,6 +1119,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5355231146689708, + "pairwise_causal_calibration_ece": 0.10733402818817206, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.208395280091001, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.23932616412639618, @@ -870,6 +1150,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5163337840632417, + "pairwise_causal_calibration_ece": 0.1979519302224726, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05628488482259619, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09099660068750381, @@ -897,6 +1181,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5157391594263406, + "pairwise_causal_calibration_ece": 0.33716773085491203, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.08837651620912834, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.020476778969168663, @@ -924,6 +1212,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.36522811458919885, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -951,6 +1243,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5116470604602193, + "pairwise_causal_calibration_ece": 0.04390849509533623, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.16064218515054388, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -978,6 +1274,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5116599733270102, + "pairwise_causal_calibration_ece": 0.05976859810156121, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.047680796780134926, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05724634602665901, @@ -1005,6 +1305,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.016455579677205734, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1032,6 +1336,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5143175918478727, + "pairwise_causal_calibration_ece": 0.12146044899072989, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27799148808206464, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13453228771686554, @@ -1059,6 +1367,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3181818181818182, + "pairwise_causal_calibration_confidence": 0.5150124877158672, + "pairwise_causal_calibration_ece": 0.19683066953404899, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.08388637942640871, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1086,6 +1398,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.518284789435327, + "pairwise_causal_calibration_ece": 0.017429496278958734, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.00913589546313831, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04792773723602295, @@ -1113,6 +1429,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5107297382813457, + "pairwise_causal_calibration_ece": 0.15358688113848856, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11986859636621072, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1140,6 +1460,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.5143045632186487, + "pairwise_causal_calibration_ece": 0.05975910867319417, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.44440232586490036, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1167,6 +1491,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5068309216451666, + "pairwise_causal_calibration_ece": 0.22111663593088093, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7913543663641995, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1194,6 +1522,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5213059897951985, + "pairwise_causal_calibration_ece": 0.09273456122376994, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7913062898533945, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.17992261052131653, @@ -1221,6 +1553,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5233637448406433, + "pairwise_causal_calibration_ece": 0.5233637448406433, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7747715144868768, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1248,6 +1584,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5094926995682197, + "pairwise_causal_calibration_ece": 0.1523498424253626, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6407352136193221, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06612107157707214, @@ -1275,6 +1615,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5117238261334832, + "pairwise_causal_calibration_ece": 0.16684760243794539, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7879470002802946, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12932512164115906, @@ -1302,6 +1646,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.1111111111111111, + "pairwise_causal_calibration_confidence": 0.5122046002000404, + "pairwise_causal_calibration_ece": 0.40109348908892933, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.8632388701651074, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1329,6 +1677,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5205501320058299, + "pairwise_causal_calibration_ece": 0.3419787034344013, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12409748155719885, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.0642656534910202, @@ -1356,6 +1708,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5125223330201494, + "pairwise_causal_calibration_ece": 0.05890623840842202, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1731915432801063, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1631268858909607, @@ -1383,6 +1739,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5160212891701115, + "pairwise_causal_calibration_ece": 0.19459271774154002, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16832709259610223, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09061422944068909, @@ -1410,6 +1770,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5125632525808054, + "pairwise_causal_calibration_ece": 0.023151033133480325, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17319751455357912, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20236456394195557, @@ -1437,6 +1801,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5145997285527485, + "pairwise_causal_calibration_ece": 0.37174258569560564, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.37573495579029975, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1464,6 +1832,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5103244375454141, + "pairwise_causal_calibration_ece": 0.2753898481688716, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.25145410930113066, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1491,6 +1863,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5144773520267778, + "pairwise_causal_calibration_ece": 0.021236933687507897, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20747832914270098, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20478898286819458, @@ -1518,6 +1894,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5167384718263315, + "pairwise_causal_calibration_ece": 0.01673847182633148, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07452586350864107, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.056939899921417236, @@ -1545,6 +1925,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.15384615384615385, + "pairwise_causal_calibration_confidence": 0.520573066371856, + "pairwise_causal_calibration_ece": 0.3667269125257021, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.335517519834288, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1572,6 +1956,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5122196492613942, + "pairwise_causal_calibration_ece": 0.5122196492613942, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.3051907131152395, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1599,6 +1987,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.24, + "pairwise_causal_calibration_confidence": 0.5125299037502397, + "pairwise_causal_calibration_ece": 0.27252990375023967, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.3049892255470116, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1626,6 +2018,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.510494424012723, + "pairwise_causal_calibration_ece": 0.1176372811555802, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14947930032054318, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1653,6 +2049,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5107784380613875, + "pairwise_causal_calibration_ece": 0.1179212952042446, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05418346476377789, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20632563531398773, @@ -1680,6 +2080,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5083113947239795, + "pairwise_causal_calibration_ece": 0.22259710900969376, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19908849658790967, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7075161337852478, @@ -1707,6 +2111,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5044609558395428, + "pairwise_causal_calibration_ece": 0.004460955839542757, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.21155860746015265, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1734,6 +2142,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.508553453802872, + "pairwise_causal_calibration_ece": 0.15141059666001483, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05257031434998286, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7209141850471497, @@ -1761,6 +2173,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5187341125070674, + "pairwise_causal_calibration_ece": 0.05269445892150404, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18633998686115366, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07200749218463898, @@ -1788,6 +2204,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.8142199631668742, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1815,6 +2235,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8928571428571429, + "pairwise_causal_calibration_confidence": 0.5143773601463993, + "pairwise_causal_calibration_ece": 0.37847978271074356, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.37155245530601344, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14892418682575226, @@ -1842,6 +2266,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5090958537532476, + "pairwise_causal_calibration_ece": 0.08052442518181907, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05550785157329158, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.030005255714058876, @@ -1869,6 +2297,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5225207590997597, + "pairwise_causal_calibration_ece": 0.3082350448140454, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.29113431369798376, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.01455958653241396, @@ -1896,6 +2328,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5214050466218352, + "pairwise_causal_calibration_ece": 0.01430923909245052, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7247859554572973, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15548823773860931, @@ -1923,6 +2359,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6111111111111112, + "pairwise_causal_calibration_confidence": 0.5206554393876835, + "pairwise_causal_calibration_ece": 0.09045567172342761, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.1780761308668144, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1950,6 +2390,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5249624623595147, + "pairwise_causal_calibration_ece": 0.09639103378808617, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.747585313191238, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.23258204758167267, @@ -1977,6 +2421,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.510137670857528, + "pairwise_causal_calibration_ece": 0.09700518628532906, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11336457853598628, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1599414199590683, @@ -2004,6 +2452,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.511453011115145, + "pairwise_causal_calibration_ece": 0.011453011115144962, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20442665184488298, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05916740372776985, @@ -2031,6 +2483,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5182750886662117, + "pairwise_causal_calibration_ece": 0.1968465172376403, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2527621054053696, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03854070231318474, @@ -2058,6 +2514,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5101349354204566, + "pairwise_causal_calibration_ece": 0.20415077886525768, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1094884748569574, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.010124921798706055, @@ -2085,6 +2545,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5180510143453494, + "pairwise_causal_calibration_ece": 0.1962346999403649, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2863411464292923, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1290181279182434, @@ -2112,6 +2576,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5125954393100116, + "pairwise_causal_calibration_ece": 0.1302617035471313, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1354556624830528, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03755499795079231, @@ -2139,6 +2607,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5224389089118766, + "pairwise_causal_calibration_ece": 0.1204182339452663, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3723174130103359, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18459641933441162, @@ -2166,6 +2638,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.525316131514424, + "pairwise_causal_calibration_ece": 0.23960184580013832, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.02532249400969805, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08514270931482315, @@ -2193,6 +2669,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5378203411218674, + "pairwise_causal_calibration_ece": 0.037820341121867385, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.28627403861735534, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04168079420924187, @@ -2220,6 +2700,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5165698214804466, + "pairwise_causal_calibration_ece": 0.12371267862330376, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.37961308921466125, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09690030664205551, @@ -2247,6 +2731,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5311319029085891, + "pairwise_causal_calibration_ece": 0.07601095423426796, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.25311843249399235, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15240873396396637, @@ -2274,6 +2762,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5179353728509533, + "pairwise_causal_calibration_ece": 0.08936394427952471, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18480216650407835, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1424257904291153, @@ -2301,6 +2793,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.16547188815803612, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2328,6 +2824,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5104711815765286, + "pairwise_causal_calibration_ece": 0.15332832443367145, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.42646156232084137, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06424945592880249, @@ -2355,6 +2855,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5168436510200018, + "pairwise_causal_calibration_ece": 0.123986508162859, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09046704020724365, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2382,6 +2886,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.520254253847887, + "pairwise_causal_calibration_ece": 0.09168282527645849, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21898086345824105, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2170039862394333, @@ -2409,6 +2917,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5117053682850884, + "pairwise_causal_calibration_ece": 0.024008917429197307, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1376102654020589, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13508938252925873, @@ -2436,6 +2948,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.5159129440129782, + "pairwise_causal_calibration_ece": 0.20822063632067045, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.46747182606481197, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2463,6 +2979,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5124017079758463, + "pairwise_causal_calibration_ece": 0.08383027940441773, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 1.0123724823810143, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2490,6 +3010,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5298422628096489, + "pairwise_causal_calibration_ece": 0.136985119952506, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.9075616877731364, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1777031123638153, @@ -2517,6 +3041,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.7046049675693343, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2544,6 +3072,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5078019052048717, + "pairwise_causal_calibration_ece": 0.06362666622369972, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6915063923618238, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05957312881946564, @@ -2571,6 +3103,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9090909090909091, + "pairwise_causal_calibration_confidence": 0.5143948483441835, + "pairwise_causal_calibration_ece": 0.3946960607467256, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.8748101531639325, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2598,6 +3134,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5246723620007077, + "pairwise_causal_calibration_ece": 0.1318152191435648, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.8761929728776295, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19969357550144196, @@ -2625,6 +3165,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5099404474997035, + "pairwise_causal_calibration_ece": 0.009940447499703464, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09482948517887341, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07914192974567413, @@ -2652,6 +3196,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5108738033410608, + "pairwise_causal_calibration_ece": 0.2251595176267751, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13594851002392472, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15427637100219727, @@ -2679,6 +3227,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5154996042105786, + "pairwise_causal_calibration_ece": 0.09164325293227849, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1359542982080382, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14092671871185303, @@ -2706,6 +3258,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5127098530524102, + "pairwise_causal_calibration_ece": 0.05871871837616116, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17781684101005288, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20639251172542572, @@ -2733,6 +3289,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.5106750580038039, + "pairwise_causal_calibration_ece": 0.19067505800380385, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.2862285662735731, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2760,6 +3320,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5072580903203541, + "pairwise_causal_calibration_ece": 0.04297237603463977, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.22345016653402755, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22683429718017578, @@ -2787,6 +3351,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.510505476566327, + "pairwise_causal_calibration_ece": 0.16806595200510155, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18071575693764783, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2086353600025177, @@ -2814,6 +3382,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5104623295849875, + "pairwise_causal_calibration_ece": 0.15331947244213034, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11645276771504434, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12235099077224731, @@ -2841,6 +3413,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5061154243359939, + "pairwise_causal_calibration_ece": 0.3510274328068632, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.20501141545015594, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2868,6 +3444,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082811179775786, + "pairwise_causal_calibration_ece": 0.07970968940615003, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.296700476651976, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2895,6 +3475,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.508536430927384, + "pairwise_causal_calibration_ece": 0.17328175089079778, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.2762512562236399, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2922,6 +3506,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.505606009084063, + "pairwise_causal_calibration_ece": 0.22166671818866435, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.1374360218451792, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2949,6 +3537,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5096046481387574, + "pairwise_causal_calibration_ece": 0.009604648138757388, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09702965995089027, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20428544282913208, @@ -2976,6 +3568,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5075633186617219, + "pairwise_causal_calibration_ece": 0.15910334800494474, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.1757025793425156, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3003,6 +3599,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.508027640450571, + "pairwise_causal_calibration_ece": 0.15088478330771388, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16787747289358793, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664259791374207, @@ -3030,6 +3630,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5086016123814964, + "pairwise_causal_calibration_ece": 0.33003018381006777, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07613754499234072, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.21979393064975739, @@ -3057,6 +3661,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5089775579590585, + "pairwise_causal_calibration_ece": 0.026736727755227174, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13367249919294005, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15220001339912415, @@ -3084,6 +3692,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.8953903319348582, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3111,6 +3723,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5151499021409406, + "pairwise_causal_calibration_ece": 0.19913581214477372, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.32264903348429946, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3138,6 +3754,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5185332008356642, + "pairwise_causal_calibration_ece": 0.017181084878621466, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1971844521448713, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09976043552160263, @@ -3165,6 +3785,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5166159575557709, + "pairwise_causal_calibration_ece": 0.019098328158514755, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16011854172306117, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03799549862742424, @@ -3192,6 +3816,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.513387181997303, + "pairwise_causal_calibration_ece": 0.22767289628301735, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6748661053169078, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10632803291082382, @@ -3219,6 +3847,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.5106422306999922, + "pairwise_causal_calibration_ece": 0.19064223069999214, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -0.06131771959154636, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3246,6 +3878,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5176593634667034, + "pairwise_causal_calibration_ece": 0.3390879348952748, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6105836341569802, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20969144999980927, @@ -3273,6 +3909,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5126630923579096, + "pairwise_causal_calibration_ece": 0.29837737807219533, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06376670876168261, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2908984124660492, @@ -3300,6 +3940,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5079024254262405, + "pairwise_causal_calibration_ece": 0.18647385399766908, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1099847634552939, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3327,6 +3971,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5249228446815363, + "pairwise_causal_calibration_ece": 0.11793429817560663, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14305325783755313, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.029790839180350304, @@ -3354,6 +4002,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5095708683932493, + "pairwise_causal_calibration_ece": 0.009570868393249277, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15950538319445437, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.007658918853849173, @@ -3381,6 +4033,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5157869727833211, + "pairwise_causal_calibration_ece": 0.19849874150239322, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18233427213439618, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3408,6 +4064,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5062707706421504, + "pairwise_causal_calibration_ece": 0.07769934207072188, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.010704936076858358, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03313249349594116, @@ -3435,6 +4095,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5280255399230291, + "pairwise_causal_calibration_ece": 0.02802553992302914, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.34974556308692006, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22729186713695526, @@ -3462,6 +4126,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5238302516808979, + "pairwise_causal_calibration_ece": 0.19045546260481638, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.049746134309109125, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08911379426717758, @@ -3489,6 +4157,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.523754237482928, + "pairwise_causal_calibration_ece": 0.19053147680278626, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1167924270620776, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.053396303206682205, @@ -3516,6 +4188,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5101656548711547, + "pairwise_causal_calibration_ece": 0.15302279772829758, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.277901316539072, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10843148082494736, @@ -3543,6 +4219,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5302602980295175, + "pairwise_causal_calibration_ece": 0.041168273399053934, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.25904253256130416, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1992303878068924, @@ -3570,6 +4250,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5194438676268386, + "pairwise_causal_calibration_ece": 0.05515815334112428, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20194937791916845, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06855231523513794, @@ -3597,6 +4281,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": -0.03896197208822921, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3624,6 +4312,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5066864884427235, + "pairwise_causal_calibration_ece": 0.0066864884427234594, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19618664619978315, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14127866923809052, @@ -3651,6 +4343,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.64, + "pairwise_causal_calibration_confidence": 0.5352559389981423, + "pairwise_causal_calibration_ece": 0.10474406100185774, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.25678792175250004, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3678,6 +4374,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5240833060484525, + "pairwise_causal_calibration_ece": 0.024083306048452546, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2126283861150165, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09991621971130371, @@ -3705,6 +4405,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5076922853795405, + "pairwise_causal_calibration_ece": 0.0076922853795404755, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.08310434328738353, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3732,6 +4436,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4090909090909091, + "pairwise_causal_calibration_confidence": 0.515496719841043, + "pairwise_causal_calibration_ece": 0.10640581075013383, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.44165993547173543, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3759,6 +4467,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.9109328188184648, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3786,6 +4498,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36, + "pairwise_causal_calibration_confidence": 0.5218845104032298, + "pairwise_causal_calibration_ece": 0.16188451040322982, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.6579069971744229, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3813,6 +4529,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5123065318085193, + "pairwise_causal_calibration_ece": 0.20197918247719504, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7487491321792448, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3840,6 +4560,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5136421948556159, + "pairwise_causal_calibration_ece": 0.013642194855615908, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6205927130038564, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06987608969211578, @@ -3867,6 +4591,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8148148148148148, + "pairwise_causal_calibration_confidence": 0.5095493338232185, + "pairwise_causal_calibration_ece": 0.3052654809915962, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.6605976642536032, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3894,6 +4622,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48148148148148145, + "pairwise_causal_calibration_confidence": 0.5122884994436931, + "pairwise_causal_calibration_ece": 0.03080701796221169, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.5028121222631348, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3921,6 +4653,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5127836243499165, + "pairwise_causal_calibration_ece": 0.13007351850722637, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11633759945598135, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06764055788516998, @@ -3948,6 +4684,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5068742634703495, + "pairwise_causal_calibration_ece": 0.11401712061320662, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12014731641743392, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19178248941898346, @@ -3975,6 +4715,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5175059157147441, + "pairwise_causal_calibration_ece": 0.017505915714744136, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13263042231741365, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1624142825603485, @@ -4002,6 +4746,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5080427209011029, + "pairwise_causal_calibration_ece": 0.17052870767032569, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19777024248040892, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5673043727874756, @@ -4029,6 +4777,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.515956688714861, + "pairwise_causal_calibration_ece": 0.0911861684279961, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.385545279380575, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.8133031129837036, @@ -4056,6 +4808,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5186359563171405, + "pairwise_causal_calibration_ece": 0.12422118654000236, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1912585754682512, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.216936856508255, @@ -4083,6 +4839,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5092055254919662, + "pairwise_causal_calibration_ece": 0.11634838263482333, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1902793460293228, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664863586425781, @@ -4110,6 +4870,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5133022716562461, + "pairwise_causal_calibration_ece": 0.40615941451338894, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12868355867896256, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18062961101531982, @@ -4137,6 +4901,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8461538461538461, + "pairwise_causal_calibration_confidence": 0.5228758201974938, + "pairwise_causal_calibration_ece": 0.32327802595635236, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.37442610603324417, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4164,6 +4932,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5098730138148706, + "pairwise_causal_calibration_ece": 0.2598730138148706, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2923757044376002, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4191,6 +4963,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5086530402095731, + "pairwise_causal_calibration_ece": 0.18722446878100168, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2974274965303909, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4211,7 +4987,18 @@ ], "schema_version": 1, "selected_lambda": 100.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, "summary": { "base_success": { "macro_by_seed": { @@ -5517,6 +6304,646 @@ }, "n": 144 }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5037706250864142, + "low": 0.4367309006782689, + "mean": 0.4704753076682902, + "n": 114, + "num_samples": 1000 + }, + "mean": 0.4704753076682902, + "per_seed": { + "1073161778/train0": 0.25, + "1073161778/train1": 0.2857142857142857, + "1073161778/train2": 0.6666666666666666, + "1113371261/train0": 0.32142857142857145, + "1113371261/train1": 0.6071428571428571, + "1113371261/train2": 0.5, + "1151410346/train0": 0.5357142857142857, + "1151410346/train1": 0.6785714285714286, + "1151410346/train2": 0.39285714285714285, + "1261930456/train0": 0.35714285714285715, + "1261930456/train1": 0.5714285714285714, + "1261930456/train2": 0.5, + "1324133122/train0": 0.5, + "1324133122/train1": 0.35714285714285715, + "1324133122/train2": 0.10714285714285714, + "1324642411/train0": 0.42857142857142855, + "1324642411/train1": 0.39285714285714285, + "1324642411/train2": 0.36, + "1375492786/train1": 0.39285714285714285, + "1375492786/train2": 0.35714285714285715, + "141984615/train0": 0.2857142857142857, + "141984615/train1": 0.6428571428571429, + "141984615/train2": 0.42857142857142855, + "1461069713/train0": 0.39285714285714285, + "1461069713/train1": 0.42857142857142855, + "1461069713/train2": 0.5357142857142857, + "1480856810/train0": 0.1111111111111111, + "1480856810/train1": 0.39285714285714285, + "1480856810/train2": 0.48148148148148145, + "1509980664/train0": 0.14285714285714285, + "1509980664/train1": 0.32, + "1509980664/train2": 0.6071428571428571, + "1522272686/train0": 0.17857142857142858, + "1522272686/train1": 0.5714285714285714, + "1522272686/train2": 0.5357142857142857, + "1549324805/train0": 0.5357142857142857, + "1549324805/train1": 0.42857142857142855, + "1549324805/train2": 0.5, + "1578990937/train0": 0.5357142857142857, + "1578990937/train1": 0.5714285714285714, + "1578990937/train2": 0.6785714285714286, + "1790188787/train0": 0.0, + "1790188787/train2": 0.7142857142857143, + "1822210471/train0": 0.35714285714285715, + "1822210471/train1": 0.5357142857142857, + "1822210471/train2": 0.5, + "1845489013/train0": 0.5, + "1845489013/train1": 0.5, + "1845489013/train2": 0.32142857142857145, + "1999022574/train0": 0.24, + "1999022574/train1": 0.6818181818181818, + "1999022574/train2": 0.32142857142857145, + "2012211287/train0": 0.5714285714285714, + "2012211287/train1": 0.39285714285714285, + "2012211287/train2": 0.7272727272727273, + "2038594191/train0": 0.5555555555555556, + "2038594191/train1": 0.6071428571428571, + "2038594191/train2": 0.5714285714285714, + "2054899126/train0": 0.6071428571428571, + "2054899126/train1": 0.5357142857142857, + "2054899126/train2": 0.2857142857142857, + "2061635888/train0": 0.17857142857142858, + "2061635888/train1": 0.5, + "2061635888/train2": 0.6428571428571429, + "211245842/train0": 0.7857142857142857, + "211245842/train1": 0.4642857142857143, + "211245842/train2": 0.6428571428571429, + "278008524/train0": 0.40909090909090906, + "278008524/train1": 0.5, + "278008524/train2": 0.42714285714285716, + "307068879/train0": 0.5, + "307068879/train1": 0.39285714285714285, + "307068879/train2": 0.5, + "412102563/train0": 0.0, + "412102563/train1": 0.42857142857142855, + "412102563/train2": 0.25, + "440577711/train0": 0.2857142857142857, + "440577711/train1": 0.42857142857142855, + "441046872/train0": 0.25, + "441046872/train1": 0.35714285714285715, + "441046872/train2": 0.17857142857142858, + "509430392/train0": 0.6785714285714286, + "509430392/train1": 0.9090909090909091, + "509430392/train2": 0.8148148148148148, + "585904130/train0": 0.7222222222222222, + "585904130/train1": 0.7142857142857143, + "585904130/train2": 0.7142857142857143, + "608016290/train0": 0.8214285714285714, + "608016290/train1": 0.5, + "608016290/train2": 0.35714285714285715, + "626486813/train0": 0.30357142857142855, + "626486813/train1": 0.26785714285714285, + "626486813/train2": 0.5892857142857143, + "636680137/train0": 0.7142857142857143, + "636680137/train1": 0.2857142857142857, + "636680137/train2": 0.7142857142857143, + "655884605/train0": 0.5714285714285714, + "655884605/train1": 0.2857142857142857, + "655884605/train2": 0.39285714285714285, + "672328262/train0": 0.6363636363636364, + "672328262/train1": 0.6111111111111112, + "672328262/train2": 0.32, + "701667486/train0": 0.15384615384615385, + "701667486/train1": 0.8571428571428571, + "701667486/train2": 0.8461538461538461, + "747260437/train0": 0.39285714285714285, + "747260437/train1": 0.35714285714285715, + "747260437/train2": 0.5, + "750448296/train0": 0.5562770562770564, + "750448296/train1": 0.4597069597069597, + "750448296/train2": 0.4577922077922078, + "767747424/train0": 0.45238095238095233, + "767747424/train1": 0.6339285714285714, + "767747424/train2": 0.5267857142857143 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.47767443592138303, + "low": 0.43998851611351614, + "mean": 0.45828770825259973, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.45828770825259973, + "per_task": { + "LiftPegUpright-v1": 0.45007215007215007, + "PickCube-v1": 0.42857142857142855, + "PullCube-v1": 0.49109454393036484, + "PushCube-v1": 0.46415614478114486, + "StackCube-v1": 0.45754427390791025 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5045185632499068, + "low": 0.44297882327733085, + "mean": 0.4736238084745547, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5155091546712853, + "low": 0.5133690823772449, + "mean": 0.5143734353718006, + "n": 114, + "num_samples": 1000 + }, + "mean": 0.5143734353718006, + "per_seed": { + "1073161778/train0": 0.507958960013907, + "1073161778/train1": 0.5083113947239795, + "1073161778/train2": 0.5075633186617219, + "1113371261/train0": 0.5160212891701115, + "1113371261/train1": 0.5154996042105786, + "1113371261/train2": 0.5175059157147441, + "1151410346/train0": 0.5144773520267778, + "1151410346/train1": 0.510505476566327, + "1151410346/train2": 0.5092055254919662, + "1261930456/train0": 0.5094926995682197, + "1261930456/train1": 0.5078019052048717, + "1261930456/train2": 0.5136421948556159, + "1324133122/train0": 0.5167384718263315, + "1324133122/train1": 0.5104623295849875, + "1324133122/train2": 0.5133022716562461, + "1324642411/train0": 0.5213059897951985, + "1324642411/train1": 0.5298422628096489, + "1324642411/train2": 0.5218845104032298, + "1375492786/train1": 0.5165698214804466, + "1375492786/train2": 0.5101656548711547, + "141984615/train0": 0.5232574572379338, + "141984615/train1": 0.5125954393100116, + "141984615/train2": 0.5062707706421504, + "1461069713/train0": 0.5163492540805769, + "1461069713/train1": 0.5090958537532476, + "1461069713/train2": 0.5185332008356642, + "1480856810/train0": 0.5122046002000404, + "1480856810/train1": 0.5246723620007077, + "1480856810/train2": 0.5122884994436931, + "1509980664/train0": 0.5145997285527485, + "1509980664/train1": 0.5106750580038039, + "1509980664/train2": 0.515956688714861, + "1522272686/train0": 0.5247878254029282, + "1522272686/train1": 0.5187341125070674, + "1522272686/train2": 0.5089775579590585, + "1549324805/train0": 0.518284789435327, + "1549324805/train1": 0.520254253847887, + "1549324805/train2": 0.5240833060484525, + "1578990937/train0": 0.5125632525808054, + "1578990937/train1": 0.5127098530524102, + "1578990937/train2": 0.5080427209011029, + "1790188787/train0": 0.5233637448406433, + "1790188787/train2": 0.5123065318085193, + "1822210471/train0": 0.5107297382813457, + "1822210471/train1": 0.5117053682850884, + "1822210471/train2": 0.5076922853795405, + "1845489013/train0": 0.5077057701570821, + "1845489013/train1": 0.511453011115145, + "1845489013/train2": 0.5079024254262405, + "1999022574/train0": 0.5125299037502397, + "1999022574/train1": 0.508536430927384, + "1999022574/train2": 0.5086530402095731, + "2012211287/train0": 0.5099995097795701, + "2012211287/train1": 0.510494424012723, + "2012211287/train2": 0.505606009084063, + "2038594191/train0": 0.5116470604602193, + "2038594191/train1": 0.5311319029085891, + "2038594191/train2": 0.5302602980295175, + "2054899126/train0": 0.5254447839705787, + "2054899126/train1": 0.5214050466218352, + "2054899126/train2": 0.513387181997303, + "2061635888/train0": 0.5205501320058299, + "2061635888/train1": 0.5099404474997035, + "2061635888/train2": 0.5127836243499165, + "211245842/train0": 0.5103244375454141, + "211245842/train1": 0.5072580903203541, + "211245842/train2": 0.5186359563171405, + "278008524/train0": 0.5150963336509506, + "278008524/train1": 0.5134906609387649, + "278008524/train2": 0.5239595156780259, + "307068879/train0": 0.5154469606567962, + "307068879/train1": 0.5107784380613875, + "307068879/train2": 0.5096046481387574, + "412102563/train0": 0.5122196492613942, + "412102563/train1": 0.5082811179775786, + "412102563/train2": 0.5098730138148706, + "440577711/train0": 0.5068309216451666, + "440577711/train1": 0.5124017079758463, + "441046872/train0": 0.5157279739172514, + "441046872/train1": 0.508553453802872, + "441046872/train2": 0.5086016123814964, + "509430392/train0": 0.5117238261334832, + "509430392/train1": 0.5143948483441835, + "509430392/train2": 0.5095493338232185, + "585904130/train0": 0.5232513345220312, + "585904130/train1": 0.5180510143453494, + "585904130/train2": 0.5157869727833211, + "608016290/train0": 0.5080911020548196, + "608016290/train1": 0.5044609558395428, + "608016290/train2": 0.508027640450571, + "626486813/train0": 0.5150490828011897, + "626486813/train1": 0.5203979238829857, + "626486813/train2": 0.5207694011186537, + "636680137/train0": 0.5163337840632417, + "636680137/train1": 0.525316131514424, + "636680137/train2": 0.5238302516808979, + "655884605/train0": 0.5125223330201494, + "655884605/train1": 0.5108738033410608, + "655884605/train2": 0.5068742634703495, + "672328262/train0": 0.5121235763357475, + "672328262/train1": 0.5206554393876835, + "672328262/train2": 0.5106422306999922, + "701667486/train0": 0.520573066371856, + "701667486/train1": 0.5061154243359939, + "701667486/train2": 0.5228758201974938, + "747260437/train0": 0.5143175918478727, + "747260437/train1": 0.5104711815765286, + "747260437/train2": 0.5066864884427235, + "750448296/train0": 0.52049588373821, + "750448296/train1": 0.518762408591936, + "750448296/train2": 0.5209887091303035, + "767747424/train0": 0.5146957447751883, + "767747424/train1": 0.5218237747620595, + "767747424/train2": 0.5165335928709553 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5169667001033001, + "low": 0.5115607084710047, + "mean": 0.5143942950558097, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5143942950558097, + "per_task": { + "LiftPegUpright-v1": 0.5092817601052971, + "PickCube-v1": 0.5174998319563514, + "PullCube-v1": 0.5173011077127652, + "PushCube-v1": 0.5152316211782679, + "StackCube-v1": 0.5126571543263667 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5161200698362212, + "low": 0.5138750589711375, + "mean": 0.515017105565867, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.16344721069471432, + "low": 0.12368520726832294, + "mean": 0.14383701062748291, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.14383701062748291, + "per_seed": { + "1073161778/train0": 0.25795896001390695, + "1073161778/train1": 0.22259710900969376, + "1073161778/train2": 0.15910334800494474, + "1113371261/train0": 0.19459271774154002, + "1113371261/train1": 0.09164325293227849, + "1113371261/train2": 0.017505915714744136, + "1151410346/train0": 0.021236933687507897, + "1151410346/train1": 0.16806595200510155, + "1151410346/train2": 0.11634838263482333, + "1261930456/train0": 0.1523498424253626, + "1261930456/train1": 0.06362666622369972, + "1261930456/train2": 0.013642194855615908, + "1324133122/train0": 0.01673847182633148, + "1324133122/train1": 0.15331947244213034, + "1324133122/train2": 0.40615941451338894, + "1324642411/train0": 0.09273456122376994, + "1324642411/train1": 0.136985119952506, + "1324642411/train2": 0.16188451040322982, + "1375492786/train0": 0.0, + "1375492786/train1": 0.12371267862330376, + "1375492786/train2": 0.15302279772829758, + "141984615/train0": 0.23754317152364812, + "141984615/train1": 0.1302617035471313, + "141984615/train2": 0.07769934207072188, + "1461069713/train0": 0.12349211122343401, + "1461069713/train1": 0.08052442518181907, + "1461069713/train2": 0.017181084878621466, + "1480856810/train0": 0.40109348908892933, + "1480856810/train1": 0.1318152191435648, + "1480856810/train2": 0.03080701796221169, + "1509980664/train0": 0.37174258569560564, + "1509980664/train1": 0.19067505800380385, + "1509980664/train2": 0.0911861684279961, + "1522272686/train0": 0.34621639683149963, + "1522272686/train1": 0.05269445892150404, + "1522272686/train2": 0.026736727755227174, + "1549324805/train0": 0.017429496278958734, + "1549324805/train1": 0.09168282527645849, + "1549324805/train2": 0.024083306048452546, + "1578990937/train0": 0.023151033133480325, + "1578990937/train1": 0.05871871837616116, + "1578990937/train2": 0.17052870767032569, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.5233637448406433, + "1790188787/train1": 0.0, + "1790188787/train2": 0.20197918247719504, + "1822210471/train0": 0.15358688113848856, + "1822210471/train1": 0.024008917429197307, + "1822210471/train2": 0.0076922853795404755, + "1845489013/train0": 0.00770577015708207, + "1845489013/train1": 0.011453011115144962, + "1845489013/train2": 0.18647385399766908, + "1999022574/train0": 0.27252990375023967, + "1999022574/train1": 0.17328175089079778, + "1999022574/train2": 0.18722446878100168, + "2012211287/train0": 0.061429061649001304, + "2012211287/train1": 0.1176372811555802, + "2012211287/train2": 0.22166671818866435, + "2038594191/train0": 0.04390849509533623, + "2038594191/train1": 0.07601095423426796, + "2038594191/train2": 0.041168273399053934, + "2054899126/train0": 0.0408490365861392, + "2054899126/train1": 0.00715461954622526, + "2054899126/train2": 0.11383644814150867, + "2061635888/train0": 0.3419787034344013, + "2061635888/train1": 0.009940447499703464, + "2061635888/train2": 0.13007351850722637, + "211245842/train0": 0.2753898481688716, + "211245842/train1": 0.04297237603463977, + "211245842/train2": 0.12422118654000236, + "278008524/train0": 0.10600542456004156, + "278008524/train1": 0.11049584722409403, + "278008524/train2": 0.20156071953702653, + "307068879/train0": 0.015446960656796183, + "307068879/train1": 0.1179212952042446, + "307068879/train2": 0.009604648138757388, + "412102563/train0": 0.5122196492613942, + "412102563/train1": 0.07970968940615003, + "412102563/train2": 0.2598730138148706, + "440577711/train0": 0.22111663593088093, + "440577711/train1": 0.08383027940441773, + "440577711/train2": 0.0, + "441046872/train0": 0.26572797391725145, + "441046872/train1": 0.15141059666001483, + "441046872/train2": 0.33003018381006777, + "509430392/train0": 0.16684760243794539, + "509430392/train1": 0.3946960607467256, + "509430392/train2": 0.3052654809915962, + "585904130/train0": 0.198970887700191, + "585904130/train1": 0.1962346999403649, + "585904130/train2": 0.19849874150239322, + "608016290/train0": 0.31333746937375184, + "608016290/train1": 0.004460955839542757, + "608016290/train2": 0.15088478330771388, + "626486813/train0": 0.2114776542297611, + "626486813/train1": 0.2525407810258429, + "626486813/train2": 0.0685163131670607, + "636680137/train0": 0.1979519302224726, + "636680137/train1": 0.23960184580013832, + "636680137/train2": 0.19045546260481638, + "655884605/train0": 0.05890623840842202, + "655884605/train1": 0.2251595176267751, + "655884605/train2": 0.11401712061320662, + "672328262/train0": 0.1242400600278889, + "672328262/train1": 0.09045567172342761, + "672328262/train2": 0.19064223069999214, + "701667486/train0": 0.3667269125257021, + "701667486/train1": 0.3510274328068632, + "701667486/train2": 0.32327802595635236, + "747260437/train0": 0.12146044899072989, + "747260437/train1": 0.15332832443367145, + "747260437/train2": 0.0066864884427234594, + "750448296/train0": 0.07562057832097581, + "750448296/train1": 0.13933427151515382, + "750448296/train2": 0.06319650133809575, + "767747424/train0": 0.12184777113177908, + "767747424/train1": 0.1792104841214887, + "767747424/train2": 0.184581523059021 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.17248101757317133, + "low": 0.13093380812931113, + "mean": 0.15338261006992773, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.15338261006992773, + "per_task": { + "LiftPegUpright-v1": 0.15994782299532878, + "PickCube-v1": 0.14188252783607694, + "PullCube-v1": 0.11385406708296895, + "PushCube-v1": 0.17122431156157183, + "StackCube-v1": 0.18000432087369214 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.16178526949462393, + "low": 0.12386864819048331, + "mean": 0.14157013997639556, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 25.141666666666666, + "low": 22.325, + "mean": 23.8625, + "n": 120, + "num_samples": 1000 + }, + "mean": 23.8625, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 18.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 28.0, + "1324642411/train2": 25.0, + "1375492786/train0": 0.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 18.0, + "1480856810/train1": 28.0, + "1480856810/train2": 27.0, + "1509980664/train0": 28.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 7.0, + "1790188787/train1": 0.0, + "1790188787/train2": 7.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 25.0, + "1999022574/train1": 22.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 28.0, + "2012211287/train2": 22.0, + "2038594191/train0": 27.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 25.0, + "278008524/train1": 28.0, + "278008524/train2": 26.5, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 7.0, + "412102563/train1": 7.0, + "412102563/train2": 28.0, + "440577711/train0": 7.0, + "440577711/train1": 7.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 28.0, + "509430392/train1": 22.0, + "509430392/train2": 27.0, + "585904130/train0": 18.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 18.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 22.0, + "672328262/train1": 18.0, + "672328262/train2": 25.0, + "701667486/train0": 13.0, + "701667486/train1": 7.0, + "701667486/train2": 13.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 26.0, + "750448296/train1": 23.0, + "750448296/train2": 26.0, + "767747424/train0": 21.0, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 26.880000000000003, + "low": 21.513333333333335, + "mean": 24.454747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 24.454747474747474, + "per_task": { + "LiftPegUpright-v1": 26.266666666666666, + "PickCube-v1": 28.0, + "PullCube-v1": 24.133333333333333, + "PushCube-v1": 19.055555555555557, + "StackCube-v1": 24.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 25.27777777777778, + "low": 22.53472222222222, + "mean": 23.958333333333332, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, "predicted_margin": { "macro_by_seed": { "ci": { @@ -7639,6 +9066,7 @@ }, "target": "success", "tau": 0.553287218862085, + "threshold_scope": "global", "weights": [ 0.2317708333333336, -0.025167241965518138, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_seed.json index 132e130f41a307f3969742569c897f9265a9b10f..e135d442920e102b33e1c85174900344bdc36b53 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_seed.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_seed.json @@ -8,6 +8,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.507958960013907, + "pairwise_causal_calibration_ece": 0.25795896001390695, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17450214530695946, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -31,6 +35,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5083113947239795, + "pairwise_causal_calibration_ece": 0.22259710900969376, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19908849658790967, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7075161337852478, @@ -54,6 +62,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5075633186617219, + "pairwise_causal_calibration_ece": 0.15910334800494474, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.1757025793425156, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -77,6 +89,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5160212891701115, + "pairwise_causal_calibration_ece": 0.19459271774154002, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16832709259610223, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09061422944068909, @@ -100,6 +116,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5154996042105786, + "pairwise_causal_calibration_ece": 0.09164325293227849, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1359542982080382, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14092671871185303, @@ -123,6 +143,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5175059157147441, + "pairwise_causal_calibration_ece": 0.017505915714744136, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13263042231741365, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1624142825603485, @@ -146,6 +170,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5144773520267778, + "pairwise_causal_calibration_ece": 0.021236933687507897, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20747832914270098, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20478898286819458, @@ -169,6 +197,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.510505476566327, + "pairwise_causal_calibration_ece": 0.16806595200510155, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18071575693764783, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2086353600025177, @@ -192,6 +224,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5092055254919662, + "pairwise_causal_calibration_ece": 0.11634838263482333, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1902793460293228, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664863586425781, @@ -215,6 +251,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5094926995682197, + "pairwise_causal_calibration_ece": 0.1523498424253626, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6407352136193221, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06612107157707214, @@ -238,6 +278,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5078019052048717, + "pairwise_causal_calibration_ece": 0.06362666622369972, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6915063923618238, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05957312881946564, @@ -261,6 +305,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5136421948556159, + "pairwise_causal_calibration_ece": 0.013642194855615908, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6205927130038564, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06987608969211578, @@ -284,6 +332,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5167384718263315, + "pairwise_causal_calibration_ece": 0.01673847182633148, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07452586350864107, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.056939899921417236, @@ -307,6 +359,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5104623295849875, + "pairwise_causal_calibration_ece": 0.15331947244213034, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11645276771504434, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12235099077224731, @@ -330,6 +386,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.10714285714285714, + "pairwise_causal_calibration_confidence": 0.5133022716562461, + "pairwise_causal_calibration_ece": 0.40615941451338894, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12868355867896256, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18062961101531982, @@ -353,6 +413,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5213059897951985, + "pairwise_causal_calibration_ece": 0.09273456122376994, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7913062898533945, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.17992261052131653, @@ -376,6 +440,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5298422628096489, + "pairwise_causal_calibration_ece": 0.136985119952506, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.9075616877731364, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1777031123638153, @@ -399,6 +467,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36, + "pairwise_causal_calibration_confidence": 0.5218845104032298, + "pairwise_causal_calibration_ece": 0.16188451040322982, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.6579069971744229, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -422,6 +494,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.36522811458919885, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -445,6 +519,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5165698214804466, + "pairwise_causal_calibration_ece": 0.12371267862330376, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.37961308921466125, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09690030664205551, @@ -468,6 +546,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5101656548711547, + "pairwise_causal_calibration_ece": 0.15302279772829758, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.277901316539072, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10843148082494736, @@ -491,6 +573,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5232574572379338, + "pairwise_causal_calibration_ece": 0.23754317152364812, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20868979202742993, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.008802552707493305, @@ -514,6 +600,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5125954393100116, + "pairwise_causal_calibration_ece": 0.1302617035471313, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1354556624830528, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03755499795079231, @@ -537,6 +627,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5062707706421504, + "pairwise_causal_calibration_ece": 0.07769934207072188, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.010704936076858358, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03313249349594116, @@ -560,6 +654,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5163492540805769, + "pairwise_causal_calibration_ece": 0.12349211122343401, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12959505268620136, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06010149046778679, @@ -583,6 +681,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5090958537532476, + "pairwise_causal_calibration_ece": 0.08052442518181907, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05550785157329158, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.030005255714058876, @@ -606,6 +708,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5185332008356642, + "pairwise_causal_calibration_ece": 0.017181084878621466, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1971844521448713, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09976043552160263, @@ -629,6 +735,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.1111111111111111, + "pairwise_causal_calibration_confidence": 0.5122046002000404, + "pairwise_causal_calibration_ece": 0.40109348908892933, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.8632388701651074, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -652,6 +762,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5246723620007077, + "pairwise_causal_calibration_ece": 0.1318152191435648, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.8761929728776295, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19969357550144196, @@ -675,6 +789,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48148148148148145, + "pairwise_causal_calibration_confidence": 0.5122884994436931, + "pairwise_causal_calibration_ece": 0.03080701796221169, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.5028121222631348, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -698,6 +816,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5145997285527485, + "pairwise_causal_calibration_ece": 0.37174258569560564, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.37573495579029975, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -721,6 +843,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.5106750580038039, + "pairwise_causal_calibration_ece": 0.19067505800380385, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.2862285662735731, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -744,6 +870,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.515956688714861, + "pairwise_causal_calibration_ece": 0.0911861684279961, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.385545279380575, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.8133031129837036, @@ -767,6 +897,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5247878254029282, + "pairwise_causal_calibration_ece": 0.34621639683149963, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14162108969262518, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16821590065956116, @@ -790,6 +924,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5187341125070674, + "pairwise_causal_calibration_ece": 0.05269445892150404, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18633998686115366, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07200749218463898, @@ -813,6 +951,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5089775579590585, + "pairwise_causal_calibration_ece": 0.026736727755227174, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13367249919294005, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15220001339912415, @@ -836,6 +978,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.518284789435327, + "pairwise_causal_calibration_ece": 0.017429496278958734, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.00913589546313831, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04792773723602295, @@ -859,6 +1005,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.520254253847887, + "pairwise_causal_calibration_ece": 0.09168282527645849, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.21898086345824105, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2170039862394333, @@ -882,6 +1032,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5240833060484525, + "pairwise_causal_calibration_ece": 0.024083306048452546, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2126283861150165, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09991621971130371, @@ -905,6 +1059,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5125632525808054, + "pairwise_causal_calibration_ece": 0.023151033133480325, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17319751455357912, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20236456394195557, @@ -928,6 +1086,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5127098530524102, + "pairwise_causal_calibration_ece": 0.05871871837616116, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17781684101005288, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20639251172542572, @@ -951,6 +1113,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5080427209011029, + "pairwise_causal_calibration_ece": 0.17052870767032569, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19777024248040892, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5673043727874756, @@ -974,6 +1140,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.7523816581368387, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -997,6 +1165,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.8142199631668742, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1020,6 +1190,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.8953903319348582, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1043,6 +1215,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5233637448406433, + "pairwise_causal_calibration_ece": 0.5233637448406433, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7747715144868768, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1066,6 +1242,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.7046049675693343, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1089,6 +1267,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5123065318085193, + "pairwise_causal_calibration_ece": 0.20197918247719504, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7487491321792448, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1112,6 +1294,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5107297382813457, + "pairwise_causal_calibration_ece": 0.15358688113848856, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11986859636621072, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1135,6 +1321,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5117053682850884, + "pairwise_causal_calibration_ece": 0.024008917429197307, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1376102654020589, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13508938252925873, @@ -1158,6 +1348,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5076922853795405, + "pairwise_causal_calibration_ece": 0.0076922853795404755, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.08310434328738353, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1181,6 +1375,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5077057701570821, + "pairwise_causal_calibration_ece": 0.00770577015708207, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13991927423910702, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02055191993713379, @@ -1204,6 +1402,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.511453011115145, + "pairwise_causal_calibration_ece": 0.011453011115144962, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20442665184488298, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05916740372776985, @@ -1227,6 +1429,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5079024254262405, + "pairwise_causal_calibration_ece": 0.18647385399766908, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1099847634552939, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1250,6 +1456,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.24, + "pairwise_causal_calibration_confidence": 0.5125299037502397, + "pairwise_causal_calibration_ece": 0.27252990375023967, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.3049892255470116, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1273,6 +1483,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.508536430927384, + "pairwise_causal_calibration_ece": 0.17328175089079778, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.2762512562236399, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1296,6 +1510,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5086530402095731, + "pairwise_causal_calibration_ece": 0.18722446878100168, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2974274965303909, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1319,6 +1537,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5099995097795701, + "pairwise_causal_calibration_ece": 0.061429061649001304, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15525138726863855, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1342,6 +1564,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.510494424012723, + "pairwise_causal_calibration_ece": 0.1176372811555802, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14947930032054318, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1365,6 +1591,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.505606009084063, + "pairwise_causal_calibration_ece": 0.22166671818866435, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.1374360218451792, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1388,6 +1618,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5116470604602193, + "pairwise_causal_calibration_ece": 0.04390849509533623, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.16064218515054388, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1411,6 +1645,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5311319029085891, + "pairwise_causal_calibration_ece": 0.07601095423426796, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.25311843249399235, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15240873396396637, @@ -1434,6 +1672,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5302602980295175, + "pairwise_causal_calibration_ece": 0.041168273399053934, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.25904253256130416, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1992303878068924, @@ -1457,6 +1699,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5254447839705787, + "pairwise_causal_calibration_ece": 0.0408490365861392, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.3415021258595635, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0780495330691338, @@ -1480,6 +1726,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5214050466218352, + "pairwise_causal_calibration_ece": 0.00715461954622526, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.4451289218076667, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0777441188693047, @@ -1503,6 +1753,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.513387181997303, + "pairwise_causal_calibration_ece": 0.11383644814150867, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.3179520666143393, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.053164016455412, @@ -1526,6 +1780,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5205501320058299, + "pairwise_causal_calibration_ece": 0.3419787034344013, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12409748155719885, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.0642656534910202, @@ -1549,6 +1807,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5099404474997035, + "pairwise_causal_calibration_ece": 0.009940447499703464, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09482948517887341, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07914192974567413, @@ -1572,6 +1834,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5127836243499165, + "pairwise_causal_calibration_ece": 0.13007351850722637, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.11633759945598135, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06764055788516998, @@ -1595,6 +1861,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.5103244375454141, + "pairwise_causal_calibration_ece": 0.2753898481688716, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.25145410930113066, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1618,6 +1888,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5072580903203541, + "pairwise_causal_calibration_ece": 0.04297237603463977, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.22345016653402755, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22683429718017578, @@ -1641,6 +1915,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5186359563171405, + "pairwise_causal_calibration_ece": 0.12422118654000236, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1912585754682512, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.216936856508255, @@ -1664,6 +1942,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.40909090909090906, + "pairwise_causal_calibration_confidence": 0.5150963336509506, + "pairwise_causal_calibration_ece": 0.10600542456004156, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.12165660849658652, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0352020561695099, @@ -1687,6 +1969,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5134906609387649, + "pairwise_causal_calibration_ece": 0.11049584722409403, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.10191580937161496, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0799707099795341, @@ -1710,6 +1996,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42714285714285716, + "pairwise_causal_calibration_confidence": 0.5239595156780259, + "pairwise_causal_calibration_ece": 0.20156071953702653, + "pairwise_causal_calibration_pairs": 26.5, "predicted_margin": 0.16027731525709132, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.1454492062330246, @@ -1733,6 +2023,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5154469606567962, + "pairwise_causal_calibration_ece": 0.015446960656796183, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12488897315305114, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20391923189163208, @@ -1756,6 +2050,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5107784380613875, + "pairwise_causal_calibration_ece": 0.1179212952042446, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05418346476377789, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20632563531398773, @@ -1779,6 +2077,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5096046481387574, + "pairwise_causal_calibration_ece": 0.009604648138757388, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09702965995089027, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20428544282913208, @@ -1802,6 +2104,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5122196492613942, + "pairwise_causal_calibration_ece": 0.5122196492613942, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.3051907131152395, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1825,6 +2131,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5082811179775786, + "pairwise_causal_calibration_ece": 0.07970968940615003, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.296700476651976, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1848,6 +2158,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5098730138148706, + "pairwise_causal_calibration_ece": 0.2598730138148706, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2923757044376002, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1871,6 +2185,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5068309216451666, + "pairwise_causal_calibration_ece": 0.22111663593088093, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.7913543663641995, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1894,6 +2212,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5124017079758463, + "pairwise_causal_calibration_ece": 0.08383027940441773, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 1.0123724823810143, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1917,6 +2239,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.9109328188184648, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1940,6 +2264,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5157279739172514, + "pairwise_causal_calibration_ece": 0.26572797391725145, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1335357706256974, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5837622880935669, @@ -1963,6 +2291,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.508553453802872, + "pairwise_causal_calibration_ece": 0.15141059666001483, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05257031434998286, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7209141850471497, @@ -1986,6 +2318,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.5086016123814964, + "pairwise_causal_calibration_ece": 0.33003018381006777, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07613754499234072, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.21979393064975739, @@ -2009,6 +2345,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5117238261334832, + "pairwise_causal_calibration_ece": 0.16684760243794539, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7879470002802946, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12932512164115906, @@ -2032,6 +2372,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.9090909090909091, + "pairwise_causal_calibration_confidence": 0.5143948483441835, + "pairwise_causal_calibration_ece": 0.3946960607467256, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.8748101531639325, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2055,6 +2399,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8148148148148148, + "pairwise_causal_calibration_confidence": 0.5095493338232185, + "pairwise_causal_calibration_ece": 0.3052654809915962, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.6605976642536032, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2078,6 +2426,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7222222222222222, + "pairwise_causal_calibration_confidence": 0.5232513345220312, + "pairwise_causal_calibration_ece": 0.198970887700191, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.16777309899666812, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2101,6 +2453,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5180510143453494, + "pairwise_causal_calibration_ece": 0.1962346999403649, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2863411464292923, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1290181279182434, @@ -2124,6 +2480,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5157869727833211, + "pairwise_causal_calibration_ece": 0.19849874150239322, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.18233427213439618, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2147,6 +2507,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8214285714285714, + "pairwise_causal_calibration_confidence": 0.5080911020548196, + "pairwise_causal_calibration_ece": 0.31333746937375184, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19628991784756858, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2170,6 +2534,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5044609558395428, + "pairwise_causal_calibration_ece": 0.004460955839542757, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.21155860746015265, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2193,6 +2561,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.508027640450571, + "pairwise_causal_calibration_ece": 0.15088478330771388, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16787747289358793, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664259791374207, @@ -2216,6 +2588,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.30357142857142855, + "pairwise_causal_calibration_confidence": 0.5150490828011897, + "pairwise_causal_calibration_ece": 0.2114776542297611, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23949765157862385, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04293955350294709, @@ -2239,6 +2615,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.26785714285714285, + "pairwise_causal_calibration_confidence": 0.5203979238829857, + "pairwise_causal_calibration_ece": 0.2525407810258429, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27194820955167665, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02655014442279935, @@ -2262,6 +2642,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5892857142857143, + "pairwise_causal_calibration_confidence": 0.5207694011186537, + "pairwise_causal_calibration_ece": 0.0685163131670607, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15158589978030715, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03389316890388727, @@ -2285,6 +2669,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5163337840632417, + "pairwise_causal_calibration_ece": 0.1979519302224726, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05628488482259619, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09099660068750381, @@ -2308,6 +2696,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.525316131514424, + "pairwise_causal_calibration_ece": 0.23960184580013832, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.02532249400969805, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08514270931482315, @@ -2331,6 +2723,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5238302516808979, + "pairwise_causal_calibration_ece": 0.19045546260481638, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.049746134309109125, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08911379426717758, @@ -2354,6 +2750,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5125223330201494, + "pairwise_causal_calibration_ece": 0.05890623840842202, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1731915432801063, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1631268858909607, @@ -2377,6 +2777,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5108738033410608, + "pairwise_causal_calibration_ece": 0.2251595176267751, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13594851002392472, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15427637100219727, @@ -2400,6 +2804,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5068742634703495, + "pairwise_causal_calibration_ece": 0.11401712061320662, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12014731641743392, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19178248941898346, @@ -2423,6 +2831,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6363636363636364, + "pairwise_causal_calibration_confidence": 0.5121235763357475, + "pairwise_causal_calibration_ece": 0.1242400600278889, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.037000828296731625, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2446,6 +2858,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6111111111111112, + "pairwise_causal_calibration_confidence": 0.5206554393876835, + "pairwise_causal_calibration_ece": 0.09045567172342761, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.1780761308668144, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2469,6 +2885,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.5106422306999922, + "pairwise_causal_calibration_ece": 0.19064223069999214, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -0.06131771959154636, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2492,6 +2912,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.15384615384615385, + "pairwise_causal_calibration_confidence": 0.520573066371856, + "pairwise_causal_calibration_ece": 0.3667269125257021, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.335517519834288, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2515,6 +2939,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5061154243359939, + "pairwise_causal_calibration_ece": 0.3510274328068632, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.20501141545015594, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2538,6 +2966,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8461538461538461, + "pairwise_causal_calibration_confidence": 0.5228758201974938, + "pairwise_causal_calibration_ece": 0.32327802595635236, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.37442610603324417, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2561,6 +2993,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5143175918478727, + "pairwise_causal_calibration_ece": 0.12146044899072989, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.27799148808206464, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13453228771686554, @@ -2584,6 +3020,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5104711815765286, + "pairwise_causal_calibration_ece": 0.15332832443367145, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.42646156232084137, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06424945592880249, @@ -2607,6 +3047,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5066864884427235, + "pairwise_causal_calibration_ece": 0.0066864884427234594, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.19618664619978315, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14127866923809052, @@ -2630,6 +3074,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5562770562770564, + "pairwise_causal_calibration_confidence": 0.52049588373821, + "pairwise_causal_calibration_ece": 0.07562057832097581, + "pairwise_causal_calibration_pairs": 26.0, "predicted_margin": 0.23349280091201208, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7655241700510184, @@ -2653,6 +3101,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4597069597069597, + "pairwise_causal_calibration_confidence": 0.518762408591936, + "pairwise_causal_calibration_ece": 0.13933427151515382, + "pairwise_causal_calibration_pairs": 23.0, "predicted_margin": 0.3415304685264087, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7756740699211756, @@ -2676,6 +3128,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.4577922077922078, + "pairwise_causal_calibration_confidence": 0.5209887091303035, + "pairwise_causal_calibration_ece": 0.06319650133809575, + "pairwise_causal_calibration_pairs": 26.0, "predicted_margin": 0.33111829215927463, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7652813941240311, @@ -2699,6 +3155,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.45238095238095233, + "pairwise_causal_calibration_confidence": 0.5146957447751883, + "pairwise_causal_calibration_ece": 0.12184777113177908, + "pairwise_causal_calibration_pairs": 21.0, "predicted_margin": 0.24227407281226254, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0471355519257486, @@ -2722,6 +3182,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.6339285714285714, + "pairwise_causal_calibration_confidence": 0.5218237747620595, + "pairwise_causal_calibration_ece": 0.1792104841214887, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.37872507049289106, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10832798760384321, @@ -2745,6 +3209,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.5267857142857143, + "pairwise_causal_calibration_confidence": 0.5165335928709553, + "pairwise_causal_calibration_ece": 0.184581523059021, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3023826194744529, "proposal_oracle_success": 0.25, "proposal_oracle_utility": 0.5676866680150852, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_task.json index 4f212776769778a1c7d29330072dc341ee1229f6..7d220144415326333ed4a9d38c862c11bd62714a 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_task.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/metrics_by_task.json @@ -8,6 +8,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.13333333333333333, + "pairwise_causal_calibration_accuracy": 0.45007215007215007, + "pairwise_causal_calibration_confidence": 0.5092817601052971, + "pairwise_causal_calibration_ece": 0.15994782299532878, + "pairwise_causal_calibration_pairs": 26.266666666666666, "predicted_margin": 0.14036877711391968, "proposal_oracle_success": 0.4666666666666667, "proposal_oracle_utility": 1.1608628551165263, @@ -31,6 +35,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5174998319563514, + "pairwise_causal_calibration_ece": 0.14188252783607694, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15387785858223962, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1308078020811081, @@ -54,6 +62,10 @@ "hidden_chart_oracle_success": 0.52, "hidden_chart_oracle_utility": 1.1467878860235214, "outcome_ptr": 0.38666666666666666, + "pairwise_causal_calibration_accuracy": 0.49109454393036484, + "pairwise_causal_calibration_confidence": 0.5173011077127652, + "pairwise_causal_calibration_ece": 0.11385406708296895, + "pairwise_causal_calibration_pairs": 24.133333333333333, "predicted_margin": 0.24822361789085187, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7346437490296861, @@ -77,6 +89,10 @@ "hidden_chart_oracle_success": 0.8333333333333334, "hidden_chart_oracle_utility": 1.7015800600250561, "outcome_ptr": 0.8333333333333334, + "pairwise_causal_calibration_accuracy": 0.46415614478114486, + "pairwise_causal_calibration_confidence": 0.5152316211782679, + "pairwise_causal_calibration_ece": 0.17122431156157183, + "pairwise_causal_calibration_pairs": 19.055555555555557, "predicted_margin": 0.7676662976993774, "proposal_oracle_success": 0.6111111111111112, "proposal_oracle_utility": 1.2712341505620215, @@ -100,6 +116,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.42424242424242425, + "pairwise_causal_calibration_accuracy": 0.45754427390791025, + "pairwise_causal_calibration_confidence": 0.5126571543263667, + "pairwise_causal_calibration_ece": 0.18000432087369214, + "pairwise_causal_calibration_pairs": 24.818181818181817, "predicted_margin": 0.21363471320190414, "proposal_oracle_success": 0.36363636363636365, "proposal_oracle_utility": 0.8693077586817018, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/table.tex index ba131b82e2fa76102f7cfacd07b813d78f5d3425..326935dcc691ebb1f64866f0c030ccb0908f60c5 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/table.tex +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/table.tex @@ -1,8 +1,8 @@ % Auto-generated by scripts/eval_learned_dominance_selector.py -\begin{tabular}{lrrrrrrrr} +\begin{tabular}{lrrrrrrrrr} \toprule -Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ \midrule -144 & 0.1806 & 0.8194 & 0.3333 & 0.3403 & 0.3819 & 0.4167 & 0.3681 & 0.1458 \\ +144 & 0.1806 & 0.8194 & 0.3333 & 0.3403 & 0.3819 & 0.4167 & 0.3681 & 0.1458 & 0.0392 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/train.log index fc93381e291dbb77608ecca6871622d091514b33..39caa92742dff2906bdda1c6d0dc176ed4eebe9a 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/train.log +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_success_tanh_train_to_test/train.log @@ -1,4 +1,5 @@ trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise selected_lambda=100.0 tau=0.553287 calibration_input=runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/command.txt index ba76ac45edbf77836f7c0ac191d713c5b8226982..4102d69c6d2d41fb8d062b56cbee646596698785 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/command.txt +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/command.txt @@ -1 +1 @@ -python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test --feature-set context_tangent --target utility_margin --k 8 --bootstrap-samples 1000 +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_tanh_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test --feature-set context_tangent --target utility_margin --k 8 --bootstrap-samples 1000 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/config.yaml index 0f3fae5c5472254480e25b04c27b8cd1cfeec07c..b229c6c62308ee376c051b6664ec13632b4348ed 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/config.yaml +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/config.yaml @@ -5,7 +5,14 @@ checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt eval_input: runs/ctt_base_context_obs_test_tanh_rollout_comparison/combined_measured_candidates.json eval_target_index: data/cil_charts_rgb_refs/test/index.json feature_set: context_tangent +fit_objective: pointwise k: 8 +no_markdown_report: True out_dir: runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test +pairwise_weight: 1.0 ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: None +success_bonus: 1.0 target: utility_margin +threshold_scope: global diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics.json index 89781472cd9438ba33f1ab550ef9142d5e2c52ca..303b523b139ad5ef7de02e02aa9adabd561fa05b 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics.json @@ -26,6 +26,10 @@ "hidden_chart_oracle_success": 0.6597222222222222, "hidden_chart_oracle_utility": 1.4150809271054134, "outcome_ptr": 0.3773148148148148, + "pairwise_causal_calibration_accuracy": 0.5345002023472278, + "pairwise_causal_calibration_confidence": 0.5477755611071591, + "pairwise_causal_calibration_ece": 0.013275358759931122, + "pairwise_causal_calibration_pairs": 9884.0, "proposal_oracle_success": 0.36342592592592593, "proposal_oracle_utility": 0.8099956938270751, "selected_success": 0.4027777777777778, @@ -49,6 +53,10 @@ "hidden_chart_oracle_success": 0.7291666666666666, "hidden_chart_oracle_utility": 1.5183161981403828, "outcome_ptr": 0.4166666666666667, + "pairwise_causal_calibration_accuracy": 0.4994202898550725, + "pairwise_causal_calibration_confidence": 0.5526392562637978, + "pairwise_causal_calibration_ece": 0.053218966408725234, + "pairwise_causal_calibration_pairs": 3450.0, "proposal_oracle_success": 0.3819444444444444, "proposal_oracle_utility": 0.8643959594224321, "selected_success": 0.3472222222222222, @@ -305,11 +313,203 @@ 0.8193685011254338, 0.784874822648645 ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 3456, + "num_fit_rows": 3456, + "num_pairwise_rows": 0, + "num_pointwise_rows": 3456, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", "k": 8, "num_calibration_candidates": 3456, "num_calibration_rows": 432, "num_eval_candidates": 1152, "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5345002023472278, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.013883233770861092, + "accuracy": 0.5241534988713318, + "confidence": 0.5380367326421929, + "count": 8860, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.007767948613708775, + "accuracy": 0.622, + "confidence": 0.6297679486137088, + "count": 1000, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.018343589984215147, + "accuracy": 0.7083333333333334, + "confidence": 0.7266769233175485, + "count": 24, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.013275358759931122, + "mean_confidence": 0.5477755611071591, + "n_bins": 10, + "num_pairs": 9884, + "num_rows": 432 + }, + "eval": { + "accuracy": 0.4994202898550725, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.04913055243637665, + "accuracy": 0.490540989047461, + "confidence": 0.5396715414838377, + "count": 3013, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.06754818876369717, + "accuracy": 0.5634517766497462, + "confidence": 0.6309999654134434, + "count": 394, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.20348876178388964, + "accuracy": 0.5365853658536586, + "confidence": 0.7400741276375482, + "count": 41, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.3090270066315306, + "accuracy": 0.5, + "confidence": 0.8090270066315306, + "count": 2, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.053218966408725234, + "mean_confidence": 0.5526392562637978, + "n_bins": 10, + "num_pairs": 3450, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, "report_type": "learned_dominance_selector_eval", "ridge_lambdas": [ 0.0, @@ -330,6 +530,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5282645860999182, + "pairwise_causal_calibration_ece": 0.007449699614367478, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.24046204109511743, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -357,6 +561,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5379855677606268, + "pairwise_causal_calibration_ece": 0.14512842490348393, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06725067375153992, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20391923189163208, @@ -384,6 +592,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5309902650109273, + "pairwise_causal_calibration_ece": 0.20956169358235582, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3519815296574341, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -411,6 +623,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5370595008141429, + "pairwise_causal_calibration_ece": 0.1772262134715715, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.31041922585570225, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -438,6 +654,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5512407286357316, + "pairwise_causal_calibration_ece": 0.30124072863573154, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07293424222875167, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5837622880935669, @@ -465,6 +685,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5693767590614354, + "pairwise_causal_calibration_ece": 0.14490895522427885, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05979507637634163, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16821590065956116, @@ -492,6 +716,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 1.0648358478225954, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -519,6 +747,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": -0.9132606733088686, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -546,6 +778,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5578235857861482, + "pairwise_causal_calibration_ece": 0.20068072864329112, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.018251630675656938, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06010149046778679, @@ -573,6 +809,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5628244973622173, + "pairwise_causal_calibration_ece": 0.09853878307650306, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.990233441085783, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.012636144645512104, @@ -600,6 +840,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5684019575451547, + "pairwise_causal_calibration_ece": 0.038740899597702605, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.9304612900205091, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15609906613826752, @@ -627,6 +871,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5637529588592934, + "pairwise_causal_calibration_ece": 0.1635197684134338, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -1.025797948011912, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -654,6 +902,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5382421898548486, + "pairwise_causal_calibration_ece": 0.1760435244308657, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3738608156205724, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16806542873382568, @@ -681,6 +933,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5808251582672191, + "pairwise_causal_calibration_ece": 0.06203198458992348, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.5604466520161834, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07040411233901978, @@ -708,6 +964,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.532640041242949, + "pairwise_causal_calibration_ece": 0.03264004124294895, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.3327016949699648, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02055191993713379, @@ -735,6 +995,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5606035262086538, + "pairwise_causal_calibration_ece": 0.29653933093420337, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.13186700575871213, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07324296236038208, @@ -762,6 +1026,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5451804712620346, + "pairwise_causal_calibration_ece": 0.09767667159510829, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.6599900589222745, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -789,6 +1057,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8888888888888888, + "pairwise_causal_calibration_confidence": 0.578858532510926, + "pairwise_causal_calibration_ece": 0.3100303563779629, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.4904640951760697, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -816,6 +1088,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5484627090361573, + "pairwise_causal_calibration_ece": 0.13010871953527126, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.09959666831852276, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.008802552707493305, @@ -843,6 +1119,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7857142857142857, + "pairwise_causal_calibration_confidence": 0.6040955897282446, + "pairwise_causal_calibration_ece": 0.1816186959860412, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.11860801847831087, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.23932616412639618, @@ -870,6 +1150,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5721923655321549, + "pairwise_causal_calibration_ece": 0.10637906303927375, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.03794194536101764, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09099660068750381, @@ -897,6 +1181,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5384474168833001, + "pairwise_causal_calibration_ece": 0.2884474168833001, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.29021298126923367, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.020476778969168663, @@ -924,6 +1212,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": -0.776948112063009, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -951,6 +1243,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2962962962962963, + "pairwise_causal_calibration_confidence": 0.5608354799929481, + "pairwise_causal_calibration_ece": 0.2645391836966521, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": -0.5762588336854821, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -978,6 +1274,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5840929634713115, + "pairwise_causal_calibration_ece": 0.21868481145242302, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.43952036453910737, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05724634602665901, @@ -1005,6 +1305,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.43093626982937056, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1032,6 +1336,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5377670577732301, + "pairwise_causal_calibration_ece": 0.10919562920180154, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8883041625225123, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13453228771686554, @@ -1059,6 +1367,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5454545454545454, + "pairwise_causal_calibration_confidence": 0.5583157186609514, + "pairwise_causal_calibration_ece": 0.14426095576686915, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.7081361184101117, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1086,6 +1398,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.6255810274188203, + "pairwise_causal_calibration_ece": 0.44700959884739166, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09636113864335544, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04792773723602295, @@ -1113,6 +1429,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5341278222988358, + "pairwise_causal_calibration_ece": 0.39127067944169297, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8104468832032641, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1140,6 +1460,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.36363636363636365, + "pairwise_causal_calibration_confidence": 0.5466553633573448, + "pairwise_causal_calibration_ece": 0.1830189997209811, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -0.41371274326291996, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1167,6 +1491,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5338564550039056, + "pairwise_causal_calibration_ece": 0.18042925928180875, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.3894302264698374, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1194,6 +1522,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5348257327159318, + "pairwise_causal_calibration_ece": 0.21339716128736036, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14065920464357806, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.17992261052131653, @@ -1221,6 +1553,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5305521535722436, + "pairwise_causal_calibration_ece": 0.38769501071510076, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.5149820474599572, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1248,6 +1584,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5275449081290765, + "pairwise_causal_calibration_ece": 0.008169377585209192, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.36400380003625826, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06612107157707214, @@ -1275,6 +1615,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5407574829724758, + "pairwise_causal_calibration_ece": 0.2193289115439045, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17166419979694275, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12932512164115906, @@ -1302,6 +1646,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6111111111111112, + "pairwise_causal_calibration_confidence": 0.538509869193624, + "pairwise_causal_calibration_ece": 0.07260124191748718, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.354808689764562, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1329,6 +1677,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5520161153108828, + "pairwise_causal_calibration_ece": 0.15915897245373997, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.19717258735290144, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.0642656534910202, @@ -1356,6 +1708,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5604641902306301, + "pairwise_causal_calibration_ece": 0.3104641902306301, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.15358387794499595, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1631268858909607, @@ -1383,6 +1739,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.560640845380424, + "pairwise_causal_calibration_ece": 0.1320694168089955, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.12619430755179695, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09061422944068909, @@ -1410,6 +1770,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5516629475083279, + "pairwise_causal_calibration_ece": 0.09119419534881495, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1237014560521208, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20236456394195557, @@ -1437,6 +1801,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5371410022092942, + "pairwise_causal_calibration_ece": 0.2514267164950085, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.08950580505034617, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1464,6 +1832,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5321100598252265, + "pairwise_causal_calibration_ece": 0.21788994017477348, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.05497412325562584, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1491,6 +1863,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5443579041048906, + "pairwise_causal_calibration_ece": 0.09849923875225228, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.085547410463293, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20478898286819458, @@ -1518,6 +1894,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5534948702896627, + "pairwise_causal_calibration_ece": 0.09144035987822766, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.03825016633859546, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.056939899921417236, @@ -1545,6 +1925,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.15384615384615385, + "pairwise_causal_calibration_confidence": 0.5335471693933942, + "pairwise_causal_calibration_ece": 0.37970101554724034, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": -0.028504930037173276, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1572,6 +1956,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5396773133557902, + "pairwise_causal_calibration_ece": 0.3968201704986473, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": -0.028276685237264176, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1599,6 +1987,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5400241860462067, + "pairwise_causal_calibration_ece": 0.1400241860462067, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -0.20399983727728455, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1626,6 +2018,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5465206862450168, + "pairwise_causal_calibration_ece": 0.153663543387874, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.38743390634925057, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1653,6 +2049,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5409161288428294, + "pairwise_causal_calibration_ece": 0.0052018431285436595, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.12159063847157181, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20632563531398773, @@ -1680,6 +2080,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5312494667075588, + "pairwise_causal_calibration_ece": 0.2455351809932731, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.39764336972690484, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7075161337852478, @@ -1707,6 +2111,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7777777777777778, + "pairwise_causal_calibration_confidence": 0.5246584653949342, + "pairwise_causal_calibration_ece": 0.2531193123828436, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.3487682967739444, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1734,6 +2142,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5413040891559014, + "pairwise_causal_calibration_ece": 0.07701837487018723, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.23111886132096565, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7209141850471497, @@ -1761,6 +2173,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5424502025658465, + "pairwise_causal_calibration_ece": 0.17183551171986777, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.15333315788417934, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07200749218463898, @@ -1788,6 +2204,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 1.5975492588263456, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1815,6 +2235,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5939854700350101, + "pairwise_causal_calibration_ece": 0.2631573871078471, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.6701379843019395, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14892418682575226, @@ -1842,6 +2266,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5530350088012301, + "pairwise_causal_calibration_ece": 0.06934714704558556, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3010944040520732, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.030005255714058876, @@ -1869,6 +2297,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5582955228728325, + "pairwise_causal_calibration_ece": 0.15599019141288176, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.4910102929178923, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.01455958653241396, @@ -1896,6 +2328,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5558201939371314, + "pairwise_causal_calibration_ece": 0.19867733679427435, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 1.2261016076527134, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15548823773860931, @@ -1923,6 +2359,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5477052910440786, + "pairwise_causal_calibration_ece": 0.031249234750163023, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.8360889673274354, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1950,6 +2390,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5351361778899372, + "pairwise_causal_calibration_ece": 0.03513617788993717, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.34258692176941946, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.23258204758167267, @@ -1977,6 +2421,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5434840545701961, + "pairwise_causal_calibration_ece": 0.15155786965593382, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8183570003839535, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1599414199590683, @@ -2004,6 +2452,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5442409997204586, + "pairwise_causal_calibration_ece": 0.044240999720458585, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.924089765000767, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05916740372776985, @@ -2031,6 +2483,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5575489905377642, + "pairwise_causal_calibration_ece": 0.19245100946223578, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.40875836795636583, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03854070231318474, @@ -2058,6 +2514,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8571428571428571, + "pairwise_causal_calibration_confidence": 0.5377569565078256, + "pairwise_causal_calibration_ece": 0.31938590063503147, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.2707099487147555, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.010124921798706055, @@ -2085,6 +2545,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5814058299730703, + "pairwise_causal_calibration_ece": 0.2757519706799789, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3658610882605051, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1290181279182434, @@ -2112,6 +2576,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5473205691724237, + "pairwise_causal_calibration_ece": 0.404463426315281, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.0898541617821001, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03755499795079231, @@ -2139,6 +2607,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5782262082924914, + "pairwise_causal_calibration_ece": 0.13605950599322297, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.12544074263761762, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18459641933441162, @@ -2166,6 +2638,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5609171219162961, + "pairwise_causal_calibration_ece": 0.3466314076305817, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.6546217738509432, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08514270931482315, @@ -2193,6 +2669,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.6056582076822431, + "pairwise_causal_calibration_ece": 0.3605910854700173, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7293758358393465, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04168079420924187, @@ -2220,6 +2700,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.570269114426804, + "pairwise_causal_calibration_ece": 0.1083023141446248, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.545927835060218, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09690030664205551, @@ -2247,6 +2731,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.6203618656848787, + "pairwise_causal_calibration_ece": 0.10965625435899745, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16597176172473033, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15240873396396637, @@ -2274,6 +2762,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5508381952618349, + "pairwise_causal_calibration_ece": 0.40798105240469207, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.024605372194202335, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1424257904291153, @@ -2301,6 +2793,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.4227329518696401, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2328,6 +2824,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5218357892576965, + "pairwise_causal_calibration_ece": 0.23612150354341077, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06722625040385721, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06424945592880249, @@ -2355,6 +2855,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5493168238068512, + "pairwise_causal_calibration_ece": 0.08503110952113695, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.4709916376837542, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2382,6 +2886,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.6115126585392462, + "pairwise_causal_calibration_ece": 0.29269130440161606, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.47555486670250297, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2170039862394333, @@ -2409,6 +2917,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5532028913257324, + "pairwise_causal_calibration_ece": 0.1960600341828752, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.2607262049073076, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13508938252925873, @@ -2436,6 +2948,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3076923076923077, + "pairwise_causal_calibration_confidence": 0.5685512118439416, + "pairwise_causal_calibration_ece": 0.26085890415163393, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": -0.4596799830828391, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2463,6 +2979,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5208261940877649, + "pairwise_causal_calibration_ece": 0.09225476551633632, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 1.014143432539303, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2490,6 +3010,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5735604901200758, + "pairwise_causal_calibration_ece": 0.5735604901200759, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.5168769022037165, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1777031123638153, @@ -2517,6 +3041,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.6458710229318597, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2544,6 +3072,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5247533765670416, + "pairwise_causal_calibration_ece": 0.024753376567041574, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.5835644081534266, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05957312881946564, @@ -2571,6 +3103,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.546796986795064, + "pairwise_causal_calibration_ece": 0.09225153224960955, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.48034283256742505, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2598,6 +3134,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5278308540547042, + "pairwise_causal_calibration_ece": 0.007883431659581497, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7918088963651103, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19969357550144196, @@ -2625,6 +3165,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5394597469948407, + "pairwise_causal_calibration_ece": 0.18231688985198352, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.1554642588014075, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07914192974567413, @@ -2652,6 +3196,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5426131615965754, + "pairwise_causal_calibration_ece": 0.18547030445371832, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.0363134192418226, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15427637100219727, @@ -2679,6 +3227,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5474740976462775, + "pairwise_causal_calibration_ece": 0.04003458353073984, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.1452439890144992, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14092671871185303, @@ -2706,6 +3258,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5443838247675006, + "pairwise_causal_calibration_ece": 0.08009811048178629, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2168280144386755, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20639251172542572, @@ -2733,6 +3289,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.540557667272934, + "pairwise_causal_calibration_ece": 0.22055766727293397, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -0.11735928488112762, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2760,6 +3320,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5386477693611458, + "pairwise_causal_calibration_ece": 0.06849508778171132, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06696359763164766, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22683429718017578, @@ -2787,6 +3351,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5283550110455164, + "pairwise_causal_calibration_ece": 0.07878784609734069, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14533366355944144, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2086353600025177, @@ -2814,6 +3382,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5540812986782463, + "pairwise_causal_calibration_ece": 0.06348258962394185, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23868082921035283, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12235099077224731, @@ -2841,6 +3413,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5253399115848963, + "pairwise_causal_calibration_ece": 0.18894580270081796, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": -0.020078414995170456, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2868,6 +3444,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5432008668839304, + "pairwise_causal_calibration_ece": 0.4003437240267876, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": -0.019469877485358084, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2895,6 +3475,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.09090909090909091, + "pairwise_causal_calibration_confidence": 0.5425459496345943, + "pairwise_causal_calibration_ece": 0.4516368587255034, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -0.17081395910193434, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2922,6 +3506,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.5211029218850123, + "pairwise_causal_calibration_ece": 0.16071525993316949, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -0.2043166539714416, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2949,6 +3537,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5381950347996567, + "pairwise_causal_calibration_ece": 0.07390932051394239, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.009308647478724175, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20428544282913208, @@ -2976,6 +3568,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5263532177022577, + "pairwise_causal_calibration_ece": 0.1403134489644089, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.2968451696979535, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3003,6 +3599,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.531822276162183, + "pairwise_causal_calibration_ece": 0.14674915240924558, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.27029964954065266, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664259791374207, @@ -3030,6 +3630,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.540704369111681, + "pairwise_causal_calibration_ece": 0.32641865482596666, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.03744323962046081, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.21979393064975739, @@ -3057,6 +3661,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5476770670178449, + "pairwise_causal_calibration_ece": 0.22624849558927332, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.03361616740919962, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15220001339912415, @@ -3084,6 +3692,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 1.207289885399395, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3111,6 +3723,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5515298890377467, + "pairwise_causal_calibration_ece": 0.1984701109622532, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.0478789592129227, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3138,6 +3754,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5614326721702952, + "pairwise_causal_calibration_ece": 0.1328612435988666, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3703409259619671, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09976043552160263, @@ -3165,6 +3785,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.25342071056365967, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5644661008458066, + "pairwise_causal_calibration_ece": 0.13589467227437815, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.9960940495855859, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03799549862742424, @@ -3192,6 +3816,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5493896182667859, + "pairwise_causal_calibration_ece": 0.22796104683821444, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.44386459699494596, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10632803291082382, @@ -3219,6 +3847,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.28, + "pairwise_causal_calibration_confidence": 0.5674349488511746, + "pairwise_causal_calibration_ece": 0.2874349488511746, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -1.1461498269869839, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3246,6 +3878,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2351006269454956, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5524675003071952, + "pairwise_causal_calibration_ece": 0.13486851977037254, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.6225553052998261, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20969144999980927, @@ -3273,6 +3909,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22781728208065033, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5637889774979964, + "pairwise_causal_calibration_ece": 0.060238703437913294, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.5952012854760782, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2908984124660492, @@ -3300,6 +3940,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5252992215396988, + "pairwise_causal_calibration_ece": 0.16815636439684162, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.1465497409461487, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3327,6 +3971,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.07774405181407928, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5441070188558171, + "pairwise_causal_calibration_ece": 0.20589298114418275, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.5910645961070223, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.029790839180350304, @@ -3354,6 +4002,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.26271408796310425, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5573829552035727, + "pairwise_causal_calibration_ece": 0.16522183114596983, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.9322959171397741, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.007658918853849173, @@ -3381,6 +4033,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5524481604069741, + "pairwise_causal_calibration_ece": 0.10741878856042494, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.6681680694969778, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3408,6 +4064,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5372758168958769, + "pairwise_causal_calibration_ece": 0.07299010261016259, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.14834785410771745, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03313249349594116, @@ -3435,6 +4095,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.6015729041919936, + "pairwise_causal_calibration_ece": 0.08906964392609457, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.028939895820333822, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22729186713695526, @@ -3462,6 +4126,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.558168129226387, + "pairwise_causal_calibration_ece": 0.022453843512101317, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.11274864478447066, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08911379426717758, @@ -3489,6 +4157,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.19710752367973328, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5519709927704145, + "pairwise_causal_calibration_ece": 0.19802900722958555, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15236517025667423, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.053396303206682205, @@ -3516,6 +4188,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5799077876470549, + "pairwise_causal_calibration_ece": 0.07990778764705499, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8143684373570463, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10843148082494736, @@ -3543,6 +4219,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.6099652051397296, + "pairwise_causal_calibration_ece": 0.08888760388581995, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13481165467333578, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1992303878068924, @@ -3570,6 +4250,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.22398948669433594, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5864156675212693, + "pairwise_causal_calibration_ece": 0.09556743888350631, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.19885536088086936, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06855231523513794, @@ -3597,6 +4281,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.00809322973959914, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3624,6 +4312,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5250941868705256, + "pairwise_causal_calibration_ece": 0.15347724170090304, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.9511918142089633, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14127866923809052, @@ -3651,6 +4343,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.88, + "pairwise_causal_calibration_confidence": 0.576434075269569, + "pairwise_causal_calibration_ece": 0.3035659247304309, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.5701797176121896, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3678,6 +4374,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.610003269112678, + "pairwise_causal_calibration_ece": 0.08182951106725747, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15222653691183596, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09991621971130371, @@ -3705,6 +4405,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5398883636934051, + "pairwise_causal_calibration_ece": 0.21845979226483364, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.7603923552010273, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3732,6 +4436,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5672660099315666, + "pairwise_causal_calibration_ece": 0.0672660099315666, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -0.8556675714722857, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3759,6 +4467,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.8439492969251245, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3786,6 +4498,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.56, + "pairwise_causal_calibration_confidence": 0.5437133949277492, + "pairwise_causal_calibration_ece": 0.033201279127124346, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.05520689035294768, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3813,6 +4529,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5141935919484805, + "pairwise_causal_calibration_ece": 0.08562216337705192, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.5287729224168903, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3840,6 +4560,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5267556650255846, + "pairwise_causal_calibration_ece": 0.06246995073987027, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.42395444307949043, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06987608969211578, @@ -3867,6 +4591,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6296296296296297, + "pairwise_causal_calibration_confidence": 0.5460667147095033, + "pairwise_causal_calibration_ece": 0.0835629149201263, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.31156668849331565, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3894,6 +4622,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5490641066742888, + "pairwise_causal_calibration_ece": 0.10461966222984442, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.08931908933026003, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -3921,6 +4653,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.546097239328519, + "pairwise_causal_calibration_ece": 0.025331332100052534, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.02416227710039781, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06764055788516998, @@ -3948,6 +4684,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5444353332736759, + "pairwise_causal_calibration_ece": 0.20556466672632412, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.013125025739580798, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19178248941898346, @@ -3975,6 +4715,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5489133106930134, + "pairwise_causal_calibration_ece": 0.26319902497872766, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.2813873904275132, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1624142825603485, @@ -4002,6 +4746,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5417842517731345, + "pairwise_causal_calibration_ece": 0.13678717679829408, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23891601330296253, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5673043727874756, @@ -4029,6 +4777,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5374866406443803, + "pairwise_causal_calibration_ece": 0.001772354930094644, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.11515799212152464, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.8133031129837036, @@ -4056,6 +4808,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.564302121709834, + "pairwise_causal_calibration_ece": 0.14998359257588048, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.04332635640223115, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.216936856508255, @@ -4083,6 +4839,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5258820853716955, + "pairwise_causal_calibration_ece": 0.02588208537169545, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1846297329460848, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664863586425781, @@ -4110,6 +4870,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5697958039405776, + "pairwise_causal_calibration_ece": 0.3197958039405776, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.0903147546551254, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18062961101531982, @@ -4137,6 +4901,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6923076923076923, + "pairwise_causal_calibration_confidence": 0.5463379229567352, + "pairwise_causal_calibration_ece": 0.14596976935095707, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.14914277579548108, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4164,6 +4932,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5483022329274447, + "pairwise_causal_calibration_ece": 0.33401651864173043, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09478211563250766, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4191,6 +4963,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5344011820922155, + "pairwise_causal_calibration_ece": 0.2129726106636441, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.07335530926221767, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -4211,7 +4987,18 @@ ], "schema_version": 1, "selected_lambda": 100.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, "summary": { "base_success": { "macro_by_seed": { @@ -5517,6 +6304,646 @@ }, "n": 144 }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5107396745554638, + "low": 0.444915242327523, + "mean": 0.47755598852090086, + "n": 114, + "num_samples": 1000 + }, + "mean": 0.47755598852090086, + "per_seed": { + "1073161778/train0": 0.32142857142857145, + "1073161778/train1": 0.2857142857142857, + "1073161778/train2": 0.6666666666666666, + "1113371261/train0": 0.42857142857142855, + "1113371261/train1": 0.5357142857142857, + "1113371261/train2": 0.2857142857142857, + "1151410346/train0": 0.6428571428571429, + "1151410346/train1": 0.6071428571428571, + "1151410346/train2": 0.5, + "1261930456/train0": 0.5357142857142857, + "1261930456/train1": 0.5, + "1261930456/train2": 0.4642857142857143, + "1324133122/train0": 0.5714285714285714, + "1324133122/train1": 0.5, + "1324133122/train2": 0.25, + "1324642411/train0": 0.32142857142857145, + "1324642411/train1": 0.0, + "1324642411/train2": 0.56, + "1375492786/train1": 0.6785714285714286, + "1375492786/train2": 0.5, + "141984615/train0": 0.6785714285714286, + "141984615/train1": 0.14285714285714285, + "141984615/train2": 0.4642857142857143, + "1461069713/train0": 0.35714285714285715, + "1461069713/train1": 0.5357142857142857, + "1461069713/train2": 0.42857142857142855, + "1480856810/train0": 0.6111111111111112, + "1480856810/train1": 0.5357142857142857, + "1480856810/train2": 0.4444444444444444, + "1509980664/train0": 0.2857142857142857, + "1509980664/train1": 0.32, + "1509980664/train2": 0.5357142857142857, + "1522272686/train0": 0.7142857142857143, + "1522272686/train1": 0.7142857142857143, + "1522272686/train2": 0.32142857142857145, + "1549324805/train0": 0.17857142857142858, + "1549324805/train1": 0.35714285714285715, + "1549324805/train2": 0.5357142857142857, + "1578990937/train0": 0.6428571428571429, + "1578990937/train1": 0.4642857142857143, + "1578990937/train2": 0.6785714285714286, + "1790188787/train0": 0.14285714285714285, + "1790188787/train2": 0.42857142857142855, + "1822210471/train0": 0.14285714285714285, + "1822210471/train1": 0.35714285714285715, + "1822210471/train2": 0.32142857142857145, + "1845489013/train0": 0.5, + "1845489013/train1": 0.5, + "1845489013/train2": 0.35714285714285715, + "1999022574/train0": 0.4, + "1999022574/train1": 0.09090909090909091, + "1999022574/train2": 0.32142857142857145, + "2012211287/train0": 0.5357142857142857, + "2012211287/train1": 0.39285714285714285, + "2012211287/train2": 0.6818181818181818, + "2038594191/train0": 0.2962962962962963, + "2038594191/train1": 0.6071428571428571, + "2038594191/train2": 0.5357142857142857, + "2054899126/train0": 0.6071428571428571, + "2054899126/train1": 0.35714285714285715, + "2054899126/train2": 0.32142857142857145, + "2061635888/train0": 0.39285714285714285, + "2061635888/train1": 0.35714285714285715, + "2061635888/train2": 0.5714285714285714, + "211245842/train0": 0.75, + "211245842/train1": 0.6071428571428571, + "211245842/train2": 0.7142857142857143, + "278008524/train0": 0.5941558441558441, + "278008524/train1": 0.5714285714285714, + "278008524/train2": 0.7435714285714285, + "307068879/train0": 0.39285714285714285, + "307068879/train1": 0.5357142857142857, + "307068879/train2": 0.4642857142857143, + "412102563/train0": 0.14285714285714285, + "412102563/train1": 0.14285714285714285, + "412102563/train2": 0.21428571428571427, + "440577711/train0": 0.7142857142857143, + "440577711/train1": 0.42857142857142855, + "441046872/train0": 0.25, + "441046872/train1": 0.4642857142857143, + "441046872/train2": 0.21428571428571427, + "509430392/train0": 0.32142857142857145, + "509430392/train1": 0.45454545454545453, + "509430392/train2": 0.6296296296296297, + "585904130/train0": 0.8888888888888888, + "585904130/train1": 0.4642857142857143, + "585904130/train2": 0.5714285714285714, + "608016290/train0": 0.7142857142857143, + "608016290/train1": 0.7777777777777778, + "608016290/train2": 0.6785714285714286, + "626486813/train0": 0.6607142857142857, + "626486813/train1": 0.7321428571428572, + "626486813/train2": 0.5892857142857143, + "636680137/train0": 0.6785714285714286, + "636680137/train1": 0.21428571428571427, + "636680137/train2": 0.5357142857142857, + "655884605/train0": 0.25, + "655884605/train1": 0.35714285714285715, + "655884605/train2": 0.75, + "672328262/train0": 0.7272727272727273, + "672328262/train1": 0.5555555555555556, + "672328262/train2": 0.28, + "701667486/train0": 0.15384615384615385, + "701667486/train1": 0.7142857142857143, + "701667486/train2": 0.6923076923076923, + "747260437/train0": 0.42857142857142855, + "747260437/train1": 0.2857142857142857, + "747260437/train2": 0.6785714285714286, + "750448296/train0": 0.5497835497835498, + "750448296/train1": 0.3882783882783883, + "750448296/train2": 0.5595238095238095, + "767747424/train0": 0.5357142857142857, + "767747424/train1": 0.6607142857142857, + "767747424/train2": 0.6964285714285714 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5457428513493643, + "low": 0.4559236344463617, + "mean": 0.49941703364854656, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49941703364854656, + "per_task": { + "LiftPegUpright-v1": 0.4917508417508418, + "PickCube-v1": 0.5833333333333334, + "PullCube-v1": 0.5280673279180743, + "PushCube-v1": 0.4432867364117365, + "StackCube-v1": 0.450646928828747 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5282241528510186, + "low": 0.4632174210532421, + "mean": 0.49605007485604485, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5537148847713698, + "low": 0.5456011239940407, + "mean": 0.5494976338759314, + "n": 114, + "num_samples": 1000 + }, + "mean": 0.5494976338759314, + "per_seed": { + "1073161778/train0": 0.5309902650109273, + "1073161778/train1": 0.5312494667075588, + "1073161778/train2": 0.5263532177022577, + "1113371261/train0": 0.560640845380424, + "1113371261/train1": 0.5474740976462775, + "1113371261/train2": 0.5489133106930134, + "1151410346/train0": 0.5443579041048906, + "1151410346/train1": 0.5283550110455164, + "1151410346/train2": 0.5258820853716955, + "1261930456/train0": 0.5275449081290765, + "1261930456/train1": 0.5247533765670416, + "1261930456/train2": 0.5267556650255846, + "1324133122/train0": 0.5534948702896627, + "1324133122/train1": 0.5540812986782463, + "1324133122/train2": 0.5697958039405776, + "1324642411/train0": 0.5348257327159318, + "1324642411/train1": 0.5735604901200758, + "1324642411/train2": 0.5437133949277492, + "1375492786/train1": 0.570269114426804, + "1375492786/train2": 0.5799077876470549, + "141984615/train0": 0.5484627090361573, + "141984615/train1": 0.5473205691724237, + "141984615/train2": 0.5372758168958769, + "1461069713/train0": 0.5578235857861482, + "1461069713/train1": 0.5530350088012301, + "1461069713/train2": 0.5614326721702952, + "1480856810/train0": 0.538509869193624, + "1480856810/train1": 0.5278308540547042, + "1480856810/train2": 0.5490641066742888, + "1509980664/train0": 0.5371410022092942, + "1509980664/train1": 0.540557667272934, + "1509980664/train2": 0.5374866406443803, + "1522272686/train0": 0.5693767590614354, + "1522272686/train1": 0.5424502025658465, + "1522272686/train2": 0.5476770670178449, + "1549324805/train0": 0.6255810274188203, + "1549324805/train1": 0.6115126585392462, + "1549324805/train2": 0.610003269112678, + "1578990937/train0": 0.5516629475083279, + "1578990937/train1": 0.5443838247675006, + "1578990937/train2": 0.5417842517731345, + "1790188787/train0": 0.5305521535722436, + "1790188787/train2": 0.5141935919484805, + "1822210471/train0": 0.5341278222988358, + "1822210471/train1": 0.5532028913257324, + "1822210471/train2": 0.5398883636934051, + "1845489013/train0": 0.532640041242949, + "1845489013/train1": 0.5442409997204586, + "1845489013/train2": 0.5252992215396988, + "1999022574/train0": 0.5400241860462067, + "1999022574/train1": 0.5425459496345943, + "1999022574/train2": 0.5344011820922155, + "2012211287/train0": 0.5282645860999182, + "2012211287/train1": 0.5465206862450168, + "2012211287/train2": 0.5211029218850123, + "2038594191/train0": 0.5608354799929481, + "2038594191/train1": 0.6203618656848787, + "2038594191/train2": 0.6099652051397296, + "2054899126/train0": 0.5684019575451547, + "2054899126/train1": 0.5558201939371314, + "2054899126/train2": 0.5493896182667859, + "2061635888/train0": 0.5520161153108828, + "2061635888/train1": 0.5394597469948407, + "2061635888/train2": 0.546097239328519, + "211245842/train0": 0.5321100598252265, + "211245842/train1": 0.5386477693611458, + "211245842/train2": 0.564302121709834, + "278008524/train0": 0.5695704384640853, + "278008524/train1": 0.5464004391885237, + "278008524/train2": 0.5701115263837827, + "307068879/train0": 0.5379855677606268, + "307068879/train1": 0.5409161288428294, + "307068879/train2": 0.5381950347996567, + "412102563/train0": 0.5396773133557902, + "412102563/train1": 0.5432008668839304, + "412102563/train2": 0.5483022329274447, + "440577711/train0": 0.5338564550039056, + "440577711/train1": 0.5208261940877649, + "441046872/train0": 0.5512407286357316, + "441046872/train1": 0.5413040891559014, + "441046872/train2": 0.540704369111681, + "509430392/train0": 0.5407574829724758, + "509430392/train1": 0.546796986795064, + "509430392/train2": 0.5460667147095033, + "585904130/train0": 0.578858532510926, + "585904130/train1": 0.5814058299730703, + "585904130/train2": 0.5524481604069741, + "608016290/train0": 0.5370595008141429, + "608016290/train1": 0.5246584653949342, + "608016290/train2": 0.531822276162183, + "626486813/train0": 0.5617140117854356, + "626486813/train1": 0.5579222567052984, + "626486813/train2": 0.5542865598508119, + "636680137/train0": 0.5721923655321549, + "636680137/train1": 0.5609171219162961, + "636680137/train2": 0.558168129226387, + "655884605/train0": 0.5604641902306301, + "655884605/train1": 0.5426131615965754, + "655884605/train2": 0.5444353332736759, + "672328262/train0": 0.5637529588592934, + "672328262/train1": 0.5477052910440786, + "672328262/train2": 0.5674349488511746, + "701667486/train0": 0.5335471693933942, + "701667486/train1": 0.5253399115848963, + "701667486/train2": 0.5463379229567352, + "747260437/train0": 0.5377670577732301, + "747260437/train1": 0.5218357892576965, + "747260437/train2": 0.5250941868705256, + "750448296/train0": 0.578281305518967, + "750448296/train1": 0.5658718717994226, + "750448296/train2": 0.5850848605482765, + "767747424/train0": 0.5406233593333943, + "767747424/train1": 0.568134203028754, + "767747424/train2": 0.5533378343297323 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5553102931747561, + "low": 0.5376258573208074, + "mean": 0.5462464570043319, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5462464570043319, + "per_task": { + "LiftPegUpright-v1": 0.5352244869552253, + "PickCube-v1": 0.5531680095483756, + "PullCube-v1": 0.5623859859314366, + "PushCube-v1": 0.5362254985310947, + "StackCube-v1": 0.5442283040555276 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5552727484438614, + "low": 0.5477087942744723, + "mean": 0.5515438388112663, + "n": 134, + "num_samples": 1000 + }, + "n": 134 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.180714330527696, + "low": 0.14076797355317144, + "mean": 0.16022162834337464, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.16022162834337464, + "per_seed": { + "1073161778/train0": 0.20956169358235582, + "1073161778/train1": 0.2455351809932731, + "1073161778/train2": 0.1403134489644089, + "1113371261/train0": 0.1320694168089955, + "1113371261/train1": 0.04003458353073984, + "1113371261/train2": 0.26319902497872766, + "1151410346/train0": 0.09849923875225228, + "1151410346/train1": 0.07878784609734069, + "1151410346/train2": 0.02588208537169545, + "1261930456/train0": 0.008169377585209192, + "1261930456/train1": 0.024753376567041574, + "1261930456/train2": 0.06246995073987027, + "1324133122/train0": 0.09144035987822766, + "1324133122/train1": 0.06348258962394185, + "1324133122/train2": 0.3197958039405776, + "1324642411/train0": 0.21339716128736036, + "1324642411/train1": 0.5735604901200759, + "1324642411/train2": 0.033201279127124346, + "1375492786/train0": 0.0, + "1375492786/train1": 0.1083023141446248, + "1375492786/train2": 0.07990778764705499, + "141984615/train0": 0.13010871953527126, + "141984615/train1": 0.404463426315281, + "141984615/train2": 0.07299010261016259, + "1461069713/train0": 0.20068072864329112, + "1461069713/train1": 0.06934714704558556, + "1461069713/train2": 0.1328612435988666, + "1480856810/train0": 0.07260124191748718, + "1480856810/train1": 0.007883431659581497, + "1480856810/train2": 0.10461966222984442, + "1509980664/train0": 0.2514267164950085, + "1509980664/train1": 0.22055766727293397, + "1509980664/train2": 0.001772354930094644, + "1522272686/train0": 0.14490895522427885, + "1522272686/train1": 0.17183551171986777, + "1522272686/train2": 0.22624849558927332, + "1549324805/train0": 0.44700959884739166, + "1549324805/train1": 0.29269130440161606, + "1549324805/train2": 0.08182951106725747, + "1578990937/train0": 0.09119419534881495, + "1578990937/train1": 0.08009811048178629, + "1578990937/train2": 0.13678717679829408, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.38769501071510076, + "1790188787/train1": 0.0, + "1790188787/train2": 0.08562216337705192, + "1822210471/train0": 0.39127067944169297, + "1822210471/train1": 0.1960600341828752, + "1822210471/train2": 0.21845979226483364, + "1845489013/train0": 0.03264004124294895, + "1845489013/train1": 0.044240999720458585, + "1845489013/train2": 0.16815636439684162, + "1999022574/train0": 0.1400241860462067, + "1999022574/train1": 0.4516368587255034, + "1999022574/train2": 0.2129726106636441, + "2012211287/train0": 0.007449699614367478, + "2012211287/train1": 0.153663543387874, + "2012211287/train2": 0.16071525993316949, + "2038594191/train0": 0.2645391836966521, + "2038594191/train1": 0.10965625435899745, + "2038594191/train2": 0.08888760388581995, + "2054899126/train0": 0.019370449798851302, + "2054899126/train1": 0.09933866839713718, + "2054899126/train2": 0.11398052341910722, + "2061635888/train0": 0.15915897245373997, + "2061635888/train1": 0.18231688985198352, + "2061635888/train2": 0.025331332100052534, + "211245842/train0": 0.21788994017477348, + "211245842/train1": 0.06849508778171132, + "211245842/train2": 0.14998359257588048, + "278008524/train0": 0.10314647017839632, + "278008524/train1": 0.11829448958853539, + "278008524/train2": 0.18190231408417212, + "307068879/train0": 0.14512842490348393, + "307068879/train1": 0.0052018431285436595, + "307068879/train2": 0.07390932051394239, + "412102563/train0": 0.3968201704986473, + "412102563/train1": 0.4003437240267876, + "412102563/train2": 0.33401651864173043, + "440577711/train0": 0.18042925928180875, + "440577711/train1": 0.09225476551633632, + "440577711/train2": 0.0, + "441046872/train0": 0.30124072863573154, + "441046872/train1": 0.07701837487018723, + "441046872/train2": 0.32641865482596666, + "509430392/train0": 0.2193289115439045, + "509430392/train1": 0.09225153224960955, + "509430392/train2": 0.0835629149201263, + "585904130/train0": 0.3100303563779629, + "585904130/train1": 0.2757519706799789, + "585904130/train2": 0.10741878856042494, + "608016290/train0": 0.1772262134715715, + "608016290/train1": 0.2531193123828436, + "608016290/train2": 0.14674915240924558, + "626486813/train0": 0.1975390570053532, + "626486813/train1": 0.17422060043755877, + "626486813/train2": 0.17089382670928044, + "636680137/train0": 0.10637906303927375, + "636680137/train1": 0.3466314076305817, + "636680137/train2": 0.022453843512101317, + "655884605/train0": 0.3104641902306301, + "655884605/train1": 0.18547030445371832, + "655884605/train2": 0.20556466672632412, + "672328262/train0": 0.1635197684134338, + "672328262/train1": 0.031249234750163023, + "672328262/train2": 0.2874349488511746, + "701667486/train0": 0.37970101554724034, + "701667486/train1": 0.18894580270081796, + "701667486/train2": 0.14596976935095707, + "747260437/train0": 0.10919562920180154, + "747260437/train1": 0.23612150354341077, + "747260437/train2": 0.15347724170090304, + "750448296/train0": 0.1944408357198151, + "750448296/train1": 0.26829982084984966, + "750448296/train2": 0.08396769758038918, + "767747424/train0": 0.14054190322731852, + "767747424/train1": 0.24456763777570825, + "767747424/train2": 0.17414736727704527 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.17804401609098255, + "low": 0.14317626806204387, + "mean": 0.16186802072277512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.16186802072277512, + "per_task": { + "LiftPegUpright-v1": 0.16155005677446432, + "PickCube-v1": 0.18099765417780664, + "PullCube-v1": 0.15891056117487964, + "PushCube-v1": 0.12454447382430739, + "StackCube-v1": 0.18333735766241757 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.17981890602497697, + "low": 0.14203169191798193, + "mean": 0.16094770301363653, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 25.141666666666666, + "low": 22.325, + "mean": 23.8625, + "n": 120, + "num_samples": 1000 + }, + "mean": 23.8625, + "per_seed": { + "1073161778/train0": 28.0, + "1073161778/train1": 28.0, + "1073161778/train2": 18.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 28.0, + "1151410346/train1": 28.0, + "1151410346/train2": 28.0, + "1261930456/train0": 28.0, + "1261930456/train1": 28.0, + "1261930456/train2": 28.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 28.0, + "1324642411/train1": 28.0, + "1324642411/train2": 25.0, + "1375492786/train0": 0.0, + "1375492786/train1": 28.0, + "1375492786/train2": 28.0, + "141984615/train0": 28.0, + "141984615/train1": 28.0, + "141984615/train2": 28.0, + "1461069713/train0": 28.0, + "1461069713/train1": 28.0, + "1461069713/train2": 28.0, + "1480856810/train0": 18.0, + "1480856810/train1": 28.0, + "1480856810/train2": 27.0, + "1509980664/train0": 28.0, + "1509980664/train1": 25.0, + "1509980664/train2": 28.0, + "1522272686/train0": 28.0, + "1522272686/train1": 28.0, + "1522272686/train2": 28.0, + "1549324805/train0": 28.0, + "1549324805/train1": 28.0, + "1549324805/train2": 28.0, + "1578990937/train0": 28.0, + "1578990937/train1": 28.0, + "1578990937/train2": 28.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 7.0, + "1790188787/train1": 0.0, + "1790188787/train2": 7.0, + "1822210471/train0": 28.0, + "1822210471/train1": 28.0, + "1822210471/train2": 28.0, + "1845489013/train0": 28.0, + "1845489013/train1": 28.0, + "1845489013/train2": 28.0, + "1999022574/train0": 25.0, + "1999022574/train1": 22.0, + "1999022574/train2": 28.0, + "2012211287/train0": 28.0, + "2012211287/train1": 28.0, + "2012211287/train2": 22.0, + "2038594191/train0": 27.0, + "2038594191/train1": 28.0, + "2038594191/train2": 28.0, + "2054899126/train0": 14.0, + "2054899126/train1": 14.0, + "2054899126/train2": 14.0, + "2061635888/train0": 28.0, + "2061635888/train1": 28.0, + "2061635888/train2": 28.0, + "211245842/train0": 28.0, + "211245842/train1": 28.0, + "211245842/train2": 28.0, + "278008524/train0": 25.0, + "278008524/train1": 28.0, + "278008524/train2": 26.5, + "307068879/train0": 28.0, + "307068879/train1": 28.0, + "307068879/train2": 28.0, + "412102563/train0": 7.0, + "412102563/train1": 7.0, + "412102563/train2": 28.0, + "440577711/train0": 7.0, + "440577711/train1": 7.0, + "440577711/train2": 0.0, + "441046872/train0": 28.0, + "441046872/train1": 28.0, + "441046872/train2": 28.0, + "509430392/train0": 28.0, + "509430392/train1": 22.0, + "509430392/train2": 27.0, + "585904130/train0": 18.0, + "585904130/train1": 28.0, + "585904130/train2": 28.0, + "608016290/train0": 28.0, + "608016290/train1": 18.0, + "608016290/train2": 28.0, + "626486813/train0": 28.0, + "626486813/train1": 28.0, + "626486813/train2": 28.0, + "636680137/train0": 28.0, + "636680137/train1": 28.0, + "636680137/train2": 28.0, + "655884605/train0": 28.0, + "655884605/train1": 28.0, + "655884605/train2": 28.0, + "672328262/train0": 22.0, + "672328262/train1": 18.0, + "672328262/train2": 25.0, + "701667486/train0": 13.0, + "701667486/train1": 7.0, + "701667486/train2": 13.0, + "747260437/train0": 28.0, + "747260437/train1": 28.0, + "747260437/train2": 28.0, + "750448296/train0": 26.0, + "750448296/train1": 23.0, + "750448296/train2": 26.0, + "767747424/train0": 21.0, + "767747424/train1": 28.0, + "767747424/train2": 28.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 26.880000000000003, + "low": 21.513333333333335, + "mean": 24.454747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 24.454747474747474, + "per_task": { + "LiftPegUpright-v1": 26.266666666666666, + "PickCube-v1": 28.0, + "PullCube-v1": 24.133333333333333, + "PushCube-v1": 19.055555555555557, + "StackCube-v1": 24.818181818181817 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 25.27777777777778, + "low": 22.53472222222222, + "mean": 23.958333333333332, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, "predicted_margin": { "macro_by_seed": { "ci": { @@ -7639,6 +9066,7 @@ }, "target": "utility_margin", "tau": 0.1353138750373699, + "threshold_scope": "global", "weights": [ -0.20962658673884535, -0.042426739033487246, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_seed.json index 03898265e6356b54f62f6f505921a27010f3122f..42056af8dd254350e116041dc7e177ad5d9f0631 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_seed.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_seed.json @@ -8,6 +8,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5309902650109273, + "pairwise_causal_calibration_ece": 0.20956169358235582, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3519815296574341, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -31,6 +35,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5312494667075588, + "pairwise_causal_calibration_ece": 0.2455351809932731, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.39764336972690484, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7075161337852478, @@ -54,6 +62,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5263532177022577, + "pairwise_causal_calibration_ece": 0.1403134489644089, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.2968451696979535, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -77,6 +89,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.560640845380424, + "pairwise_causal_calibration_ece": 0.1320694168089955, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.12619430755179695, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09061422944068909, @@ -100,6 +116,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5474740976462775, + "pairwise_causal_calibration_ece": 0.04003458353073984, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.1452439890144992, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14092671871185303, @@ -123,6 +143,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5489133106930134, + "pairwise_causal_calibration_ece": 0.26319902497872766, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.2813873904275132, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1624142825603485, @@ -146,6 +170,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5443579041048906, + "pairwise_causal_calibration_ece": 0.09849923875225228, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.085547410463293, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20478898286819458, @@ -169,6 +197,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5283550110455164, + "pairwise_causal_calibration_ece": 0.07878784609734069, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14533366355944144, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2086353600025177, @@ -192,6 +224,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5258820853716955, + "pairwise_causal_calibration_ece": 0.02588208537169545, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1846297329460848, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664863586425781, @@ -215,6 +251,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5275449081290765, + "pairwise_causal_calibration_ece": 0.008169377585209192, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.36400380003625826, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06612107157707214, @@ -238,6 +278,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5247533765670416, + "pairwise_causal_calibration_ece": 0.024753376567041574, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.5835644081534266, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05957312881946564, @@ -261,6 +305,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5267556650255846, + "pairwise_causal_calibration_ece": 0.06246995073987027, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.42395444307949043, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06987608969211578, @@ -284,6 +332,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5534948702896627, + "pairwise_causal_calibration_ece": 0.09144035987822766, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.03825016633859546, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.056939899921417236, @@ -307,6 +359,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5540812986782463, + "pairwise_causal_calibration_ece": 0.06348258962394185, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23868082921035283, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12235099077224731, @@ -330,6 +386,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5697958039405776, + "pairwise_causal_calibration_ece": 0.3197958039405776, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.0903147546551254, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.18062961101531982, @@ -353,6 +413,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5348257327159318, + "pairwise_causal_calibration_ece": 0.21339716128736036, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.14065920464357806, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.17992261052131653, @@ -376,6 +440,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5735604901200758, + "pairwise_causal_calibration_ece": 0.5735604901200759, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.5168769022037165, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1777031123638153, @@ -399,6 +467,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.56, + "pairwise_causal_calibration_confidence": 0.5437133949277492, + "pairwise_causal_calibration_ece": 0.033201279127124346, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.05520689035294768, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -422,6 +494,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": -0.776948112063009, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -445,6 +519,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.570269114426804, + "pairwise_causal_calibration_ece": 0.1083023141446248, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.545927835060218, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09690030664205551, @@ -468,6 +546,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5799077876470549, + "pairwise_causal_calibration_ece": 0.07990778764705499, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8143684373570463, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10843148082494736, @@ -491,6 +573,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5484627090361573, + "pairwise_causal_calibration_ece": 0.13010871953527126, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.09959666831852276, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.008802552707493305, @@ -514,6 +600,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5473205691724237, + "pairwise_causal_calibration_ece": 0.404463426315281, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.0898541617821001, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03755499795079231, @@ -537,6 +627,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2857772707939148, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5372758168958769, + "pairwise_causal_calibration_ece": 0.07299010261016259, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.14834785410771745, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03313249349594116, @@ -560,6 +654,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5578235857861482, + "pairwise_causal_calibration_ece": 0.20068072864329112, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.018251630675656938, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06010149046778679, @@ -583,6 +681,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5530350088012301, + "pairwise_causal_calibration_ece": 0.06934714704558556, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3010944040520732, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.030005255714058876, @@ -606,6 +708,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.1593424528837204, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5614326721702952, + "pairwise_causal_calibration_ece": 0.1328612435988666, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.3703409259619671, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09976043552160263, @@ -629,6 +735,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6111111111111112, + "pairwise_causal_calibration_confidence": 0.538509869193624, + "pairwise_causal_calibration_ece": 0.07260124191748718, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": 0.354808689764562, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -652,6 +762,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5278308540547042, + "pairwise_causal_calibration_ece": 0.007883431659581497, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.7918088963651103, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19969357550144196, @@ -675,6 +789,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.20948036015033722, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5490641066742888, + "pairwise_causal_calibration_ece": 0.10461966222984442, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.08931908933026003, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -698,6 +816,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5371410022092942, + "pairwise_causal_calibration_ece": 0.2514267164950085, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.08950580505034617, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -721,6 +843,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32, + "pairwise_causal_calibration_confidence": 0.540557667272934, + "pairwise_causal_calibration_ece": 0.22055766727293397, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -0.11735928488112762, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -744,6 +870,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5374866406443803, + "pairwise_causal_calibration_ece": 0.001772354930094644, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.11515799212152464, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.8133031129837036, @@ -767,6 +897,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5693767590614354, + "pairwise_causal_calibration_ece": 0.14490895522427885, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.05979507637634163, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.16821590065956116, @@ -790,6 +924,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5424502025658465, + "pairwise_causal_calibration_ece": 0.17183551171986777, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.15333315788417934, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07200749218463898, @@ -813,6 +951,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5476770670178449, + "pairwise_causal_calibration_ece": 0.22624849558927332, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.03361616740919962, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15220001339912415, @@ -836,6 +978,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.17857142857142858, + "pairwise_causal_calibration_confidence": 0.6255810274188203, + "pairwise_causal_calibration_ece": 0.44700959884739166, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09636113864335544, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04792773723602295, @@ -859,6 +1005,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.6115126585392462, + "pairwise_causal_calibration_ece": 0.29269130440161606, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.47555486670250297, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.2170039862394333, @@ -882,6 +1032,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2974560856819153, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.610003269112678, + "pairwise_causal_calibration_ece": 0.08182951106725747, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.15222653691183596, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09991621971130371, @@ -905,6 +1059,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6428571428571429, + "pairwise_causal_calibration_confidence": 0.5516629475083279, + "pairwise_causal_calibration_ece": 0.09119419534881495, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.1237014560521208, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20236456394195557, @@ -928,6 +1086,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5443838247675006, + "pairwise_causal_calibration_ece": 0.08009811048178629, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.2168280144386755, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20639251172542572, @@ -951,6 +1113,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5417842517731345, + "pairwise_causal_calibration_ece": 0.13678717679829408, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.23891601330296253, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5673043727874756, @@ -974,6 +1140,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 1.0648358478225954, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -997,6 +1165,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 1.5975492588263456, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1020,6 +1190,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 1.207289885399395, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1043,6 +1215,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5305521535722436, + "pairwise_causal_calibration_ece": 0.38769501071510076, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.5149820474599572, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1066,6 +1242,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.6458710229318597, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1089,6 +1267,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5141935919484805, + "pairwise_causal_calibration_ece": 0.08562216337705192, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.5287729224168903, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1112,6 +1294,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5341278222988358, + "pairwise_causal_calibration_ece": 0.39127067944169297, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8104468832032641, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1135,6 +1321,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5532028913257324, + "pairwise_causal_calibration_ece": 0.1960600341828752, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.2607262049073076, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13508938252925873, @@ -1158,6 +1348,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5398883636934051, + "pairwise_causal_calibration_ece": 0.21845979226483364, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.7603923552010273, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1181,6 +1375,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.532640041242949, + "pairwise_causal_calibration_ece": 0.03264004124294895, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.3327016949699648, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02055191993713379, @@ -1204,6 +1402,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5442409997204586, + "pairwise_causal_calibration_ece": 0.044240999720458585, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.924089765000767, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.05916740372776985, @@ -1227,6 +1429,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5252992215396988, + "pairwise_causal_calibration_ece": 0.16815636439684162, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -1.1465497409461487, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1250,6 +1456,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4, + "pairwise_causal_calibration_confidence": 0.5400241860462067, + "pairwise_causal_calibration_ece": 0.1400241860462067, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -0.20399983727728455, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1273,6 +1483,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.09090909090909091, + "pairwise_causal_calibration_confidence": 0.5425459496345943, + "pairwise_causal_calibration_ece": 0.4516368587255034, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -0.17081395910193434, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1296,6 +1510,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5344011820922155, + "pairwise_causal_calibration_ece": 0.2129726106636441, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.07335530926221767, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1319,6 +1537,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5282645860999182, + "pairwise_causal_calibration_ece": 0.007449699614367478, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.24046204109511743, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1342,6 +1564,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5465206862450168, + "pairwise_causal_calibration_ece": 0.153663543387874, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.38743390634925057, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1365,6 +1591,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.5211029218850123, + "pairwise_causal_calibration_ece": 0.16071525993316949, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -0.2043166539714416, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1388,6 +1618,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2962962962962963, + "pairwise_causal_calibration_confidence": 0.5608354799929481, + "pairwise_causal_calibration_ece": 0.2645391836966521, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": -0.5762588336854821, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1411,6 +1645,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.6203618656848787, + "pairwise_causal_calibration_ece": 0.10965625435899745, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.16597176172473033, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15240873396396637, @@ -1434,6 +1672,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.6099652051397296, + "pairwise_causal_calibration_ece": 0.08888760388581995, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.13481165467333578, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1992303878068924, @@ -1457,6 +1699,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5684019575451547, + "pairwise_causal_calibration_ece": 0.019370449798851302, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.6806987799249399, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0780495330691338, @@ -1480,6 +1726,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5558201939371314, + "pairwise_causal_calibration_ece": 0.09933866839713718, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.8244172797611767, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0777441188693047, @@ -1503,6 +1753,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5493896182667859, + "pairwise_causal_calibration_ece": 0.11398052341910722, + "pairwise_causal_calibration_pairs": 14.0, "predicted_margin": 0.22597891336727255, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.053164016455412, @@ -1526,6 +1780,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5520161153108828, + "pairwise_causal_calibration_ece": 0.15915897245373997, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.19717258735290144, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.0642656534910202, @@ -1549,6 +1807,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5394597469948407, + "pairwise_causal_calibration_ece": 0.18231688985198352, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.1554642588014075, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.07914192974567413, @@ -1572,6 +1834,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.546097239328519, + "pairwise_causal_calibration_ece": 0.025331332100052534, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.02416227710039781, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06764055788516998, @@ -1595,6 +1861,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5321100598252265, + "pairwise_causal_calibration_ece": 0.21788994017477348, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.05497412325562584, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1618,6 +1888,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6071428571428571, + "pairwise_causal_calibration_confidence": 0.5386477693611458, + "pairwise_causal_calibration_ece": 0.06849508778171132, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06696359763164766, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.22683429718017578, @@ -1641,6 +1915,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.564302121709834, + "pairwise_causal_calibration_ece": 0.14998359257588048, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.04332635640223115, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.216936856508255, @@ -1664,6 +1942,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5941558441558441, + "pairwise_causal_calibration_confidence": 0.5695704384640853, + "pairwise_causal_calibration_ece": 0.10314647017839632, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": 0.07384473319696416, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0352020561695099, @@ -1687,6 +1969,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5464004391885237, + "pairwise_causal_calibration_ece": 0.11829448958853539, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.17368268135009968, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0799707099795341, @@ -1710,6 +1996,10 @@ "hidden_chart_oracle_success": 0.5, "hidden_chart_oracle_utility": 1.1139086410403252, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7435714285714285, + "pairwise_causal_calibration_confidence": 0.5701115263837827, + "pairwise_causal_calibration_ece": 0.18190231408417212, + "pairwise_causal_calibration_pairs": 26.5, "predicted_margin": -0.012510783931944325, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.1454492062330246, @@ -1733,6 +2023,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5379855677606268, + "pairwise_causal_calibration_ece": 0.14512842490348393, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06725067375153992, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20391923189163208, @@ -1756,6 +2050,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5409161288428294, + "pairwise_causal_calibration_ece": 0.0052018431285436595, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.12159063847157181, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20632563531398773, @@ -1779,6 +2077,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5381950347996567, + "pairwise_causal_calibration_ece": 0.07390932051394239, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.009308647478724175, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.20428544282913208, @@ -1802,6 +2104,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5396773133557902, + "pairwise_causal_calibration_ece": 0.3968201704986473, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": -0.028276685237264176, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1825,6 +2131,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.14285714285714285, + "pairwise_causal_calibration_confidence": 0.5432008668839304, + "pairwise_causal_calibration_ece": 0.4003437240267876, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": -0.019469877485358084, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1848,6 +2158,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5483022329274447, + "pairwise_causal_calibration_ece": 0.33401651864173043, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.09478211563250766, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1871,6 +2185,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5338564550039056, + "pairwise_causal_calibration_ece": 0.18042925928180875, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 0.3894302264698374, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1894,6 +2212,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5208261940877649, + "pairwise_causal_calibration_ece": 0.09225476551633632, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": 1.014143432539303, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1917,6 +2239,8 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, "predicted_margin": 0.8439492969251245, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -1940,6 +2264,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5512407286357316, + "pairwise_causal_calibration_ece": 0.30124072863573154, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.07293424222875167, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5837622880935669, @@ -1963,6 +2291,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5413040891559014, + "pairwise_causal_calibration_ece": 0.07701837487018723, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.23111886132096565, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.7209141850471497, @@ -1986,6 +2318,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.540704369111681, + "pairwise_causal_calibration_ece": 0.32641865482596666, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.03744323962046081, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.21979393064975739, @@ -2009,6 +2345,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.32142857142857145, + "pairwise_causal_calibration_confidence": 0.5407574829724758, + "pairwise_causal_calibration_ece": 0.2193289115439045, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.17166419979694275, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.12932512164115906, @@ -2032,6 +2372,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45454545454545453, + "pairwise_causal_calibration_confidence": 0.546796986795064, + "pairwise_causal_calibration_ece": 0.09225153224960955, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": 0.48034283256742505, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2055,6 +2399,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6296296296296297, + "pairwise_causal_calibration_confidence": 0.5460667147095033, + "pairwise_causal_calibration_ece": 0.0835629149201263, + "pairwise_causal_calibration_pairs": 27.0, "predicted_margin": 0.31156668849331565, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2078,6 +2426,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8888888888888888, + "pairwise_causal_calibration_confidence": 0.578858532510926, + "pairwise_causal_calibration_ece": 0.3100303563779629, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.4904640951760697, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2101,6 +2453,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4642857142857143, + "pairwise_causal_calibration_confidence": 0.5814058299730703, + "pairwise_causal_calibration_ece": 0.2757519706799789, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3658610882605051, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1290181279182434, @@ -2124,6 +2480,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5524481604069741, + "pairwise_causal_calibration_ece": 0.10741878856042494, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.6681680694969778, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2147,6 +2507,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5370595008141429, + "pairwise_causal_calibration_ece": 0.1772262134715715, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.31041922585570225, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2170,6 +2534,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.7777777777777778, + "pairwise_causal_calibration_confidence": 0.5246584653949342, + "pairwise_causal_calibration_ece": 0.2531193123828436, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.3487682967739444, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2193,6 +2561,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.531822276162183, + "pairwise_causal_calibration_ece": 0.14674915240924558, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.27029964954065266, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.5664259791374207, @@ -2216,6 +2588,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6607142857142857, + "pairwise_causal_calibration_confidence": 0.5617140117854356, + "pairwise_causal_calibration_ece": 0.1975390570053532, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.42918321766353545, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.04293955350294709, @@ -2239,6 +2615,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7321428571428572, + "pairwise_causal_calibration_confidence": 0.5579222567052984, + "pairwise_causal_calibration_ece": 0.17422060043755877, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.04112596248076322, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.02655014442279935, @@ -2262,6 +2642,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.16558238118886948, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5892857142857143, + "pairwise_causal_calibration_confidence": 0.5542865598508119, + "pairwise_causal_calibration_ece": 0.17089382670928044, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.20251472673928178, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.03389316890388727, @@ -2285,6 +2669,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5721923655321549, + "pairwise_causal_calibration_ece": 0.10637906303927375, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.03794194536101764, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.09099660068750381, @@ -2308,6 +2696,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.21428571428571427, + "pairwise_causal_calibration_confidence": 0.5609171219162961, + "pairwise_causal_calibration_ece": 0.3466314076305817, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.6546217738509432, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08514270931482315, @@ -2331,6 +2723,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.17056432366371155, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.558168129226387, + "pairwise_causal_calibration_ece": 0.022453843512101317, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.11274864478447066, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.08911379426717758, @@ -2354,6 +2750,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.25, + "pairwise_causal_calibration_confidence": 0.5604641902306301, + "pairwise_causal_calibration_ece": 0.3104641902306301, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.15358387794499595, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1631268858909607, @@ -2377,6 +2777,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5426131615965754, + "pairwise_causal_calibration_ece": 0.18547030445371832, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.0363134192418226, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.15427637100219727, @@ -2400,6 +2804,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5444353332736759, + "pairwise_causal_calibration_ece": 0.20556466672632412, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.013125025739580798, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.19178248941898346, @@ -2423,6 +2831,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7272727272727273, + "pairwise_causal_calibration_confidence": 0.5637529588592934, + "pairwise_causal_calibration_ece": 0.1635197684134338, + "pairwise_causal_calibration_pairs": 22.0, "predicted_margin": -1.025797948011912, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2446,6 +2858,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5477052910440786, + "pairwise_causal_calibration_ece": 0.031249234750163023, + "pairwise_causal_calibration_pairs": 18.0, "predicted_margin": -0.8360889673274354, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2469,6 +2885,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.28, + "pairwise_causal_calibration_confidence": 0.5674349488511746, + "pairwise_causal_calibration_ece": 0.2874349488511746, + "pairwise_causal_calibration_pairs": 25.0, "predicted_margin": -1.1461498269869839, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2492,6 +2912,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.15384615384615385, + "pairwise_causal_calibration_confidence": 0.5335471693933942, + "pairwise_causal_calibration_ece": 0.37970101554724034, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": -0.028504930037173276, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2515,6 +2939,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5253399115848963, + "pairwise_causal_calibration_ece": 0.18894580270081796, + "pairwise_causal_calibration_pairs": 7.0, "predicted_margin": -0.020078414995170456, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2538,6 +2966,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6923076923076923, + "pairwise_causal_calibration_confidence": 0.5463379229567352, + "pairwise_causal_calibration_ece": 0.14596976935095707, + "pairwise_causal_calibration_pairs": 13.0, "predicted_margin": 0.14914277579548108, "proposal_oracle_success": 1.0, "proposal_oracle_utility": 2.0, @@ -2561,6 +2993,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5377670577732301, + "pairwise_causal_calibration_ece": 0.10919562920180154, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.8883041625225123, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.13453228771686554, @@ -2584,6 +3020,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2857142857142857, + "pairwise_causal_calibration_confidence": 0.5218357892576965, + "pairwise_causal_calibration_ece": 0.23612150354341077, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": 0.06722625040385721, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.06424945592880249, @@ -2607,6 +3047,10 @@ "hidden_chart_oracle_success": 0.0, "hidden_chart_oracle_utility": 0.2786632478237152, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6785714285714286, + "pairwise_causal_calibration_confidence": 0.5250941868705256, + "pairwise_causal_calibration_ece": 0.15347724170090304, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.9511918142089633, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.14127866923809052, @@ -2630,6 +3074,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5497835497835498, + "pairwise_causal_calibration_confidence": 0.578281305518967, + "pairwise_causal_calibration_ece": 0.1944408357198151, + "pairwise_causal_calibration_pairs": 26.0, "predicted_margin": -0.32394704209344605, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7655241700510184, @@ -2653,6 +3101,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3882783882783883, + "pairwise_causal_calibration_confidence": 0.5658718717994226, + "pairwise_causal_calibration_ece": 0.26829982084984966, + "pairwise_causal_calibration_pairs": 23.0, "predicted_margin": -0.10321128941700637, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7756740699211756, @@ -2676,6 +3128,10 @@ "hidden_chart_oracle_success": 0.6666666666666666, "hidden_chart_oracle_utility": 1.4079964955647786, "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5595238095238095, + "pairwise_causal_calibration_confidence": 0.5850848605482765, + "pairwise_causal_calibration_ece": 0.08396769758038918, + "pairwise_causal_calibration_pairs": 26.0, "predicted_margin": -0.3611542760578296, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7652813941240311, @@ -2699,6 +3155,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5406233593333943, + "pairwise_causal_calibration_ece": 0.14054190322731852, + "pairwise_causal_calibration_pairs": 21.0, "predicted_margin": -0.6224007244699511, "proposal_oracle_success": 0.5, "proposal_oracle_utility": 1.0471355519257486, @@ -2722,6 +3182,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.6607142857142857, + "pairwise_causal_calibration_confidence": 0.568134203028754, + "pairwise_causal_calibration_ece": 0.24456763777570825, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.21722129385198224, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.10832798760384321, @@ -2745,6 +3209,10 @@ "hidden_chart_oracle_success": 0.25, "hidden_chart_oracle_utility": 0.6737305596470833, "outcome_ptr": 0.25, + "pairwise_causal_calibration_accuracy": 0.6964285714285714, + "pairwise_causal_calibration_confidence": 0.5533378343297323, + "pairwise_causal_calibration_ece": 0.17414736727704527, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.3013136001990492, "proposal_oracle_success": 0.25, "proposal_oracle_utility": 0.5676866680150852, diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_task.json index cbe966cfdb819d899d95172a9bb8952569c3f6ff..5e0cd090ff555a0fe1e8eee5e948ac70b30f070b 100644 --- a/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_task.json +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_context_tangent_tanh_train_to_test/metrics_by_task.json @@ -8,6 +8,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.13333333333333333, + "pairwise_causal_calibration_accuracy": 0.4917508417508418, + "pairwise_causal_calibration_confidence": 0.5352244869552253, + "pairwise_causal_calibration_ece": 0.16155005677446432, + "pairwise_causal_calibration_pairs": 26.266666666666666, "predicted_margin": -0.19826283595876415, "proposal_oracle_success": 0.4666666666666667, "proposal_oracle_utility": 1.1608628551165263, @@ -31,6 +35,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5833333333333334, + "pairwise_causal_calibration_confidence": 0.5531680095483756, + "pairwise_causal_calibration_ece": 0.18099765417780664, + "pairwise_causal_calibration_pairs": 28.0, "predicted_margin": -0.019973971366212696, "proposal_oracle_success": 0.0, "proposal_oracle_utility": 0.1308078020811081, @@ -54,6 +62,10 @@ "hidden_chart_oracle_success": 0.52, "hidden_chart_oracle_utility": 1.1467878860235214, "outcome_ptr": 0.38666666666666666, + "pairwise_causal_calibration_accuracy": 0.5280673279180743, + "pairwise_causal_calibration_confidence": 0.5623859859314366, + "pairwise_causal_calibration_ece": 0.15891056117487964, + "pairwise_causal_calibration_pairs": 24.133333333333333, "predicted_margin": -0.17061686388444233, "proposal_oracle_success": 0.3333333333333333, "proposal_oracle_utility": 0.7346437490296861, @@ -77,6 +89,10 @@ "hidden_chart_oracle_success": 0.8333333333333334, "hidden_chart_oracle_utility": 1.7015800600250561, "outcome_ptr": 0.8333333333333334, + "pairwise_causal_calibration_accuracy": 0.4432867364117365, + "pairwise_causal_calibration_confidence": 0.5362254985310947, + "pairwise_causal_calibration_ece": 0.12454447382430739, + "pairwise_causal_calibration_pairs": 19.055555555555557, "predicted_margin": 0.45671805519611147, "proposal_oracle_success": 0.6111111111111112, "proposal_oracle_utility": 1.2712341505620215, @@ -100,6 +116,10 @@ "hidden_chart_oracle_success": 1.0, "hidden_chart_oracle_utility": 2.0, "outcome_ptr": 0.42424242424242425, + "pairwise_causal_calibration_accuracy": 0.450646928828747, + "pairwise_causal_calibration_confidence": 0.5442283040555276, + "pairwise_causal_calibration_ece": 0.18333735766241757, + "pairwise_causal_calibration_pairs": 24.818181818181817, "predicted_margin": -0.01023413531670979, "proposal_oracle_success": 0.36363636363636365, "proposal_oracle_utility": 0.8693077586817018,