diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..3c4cead819e7a0e8808e4887a80c8af2a0c2e3f1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test --feature-set bundle_consensus --target utility_margin --threshold-scope task --fit-objective pointwise --k 16 --bootstrap-samples 100 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..becb8aadab83300b5ae19ec6343ef2543ac11050 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 100 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: bundle_consensus +fit_objective: pointwise +k: 16 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs_obj +source_index: data/cil_charts_rgb_refs/train/index.json +success_bonus: 1.0 +target: utility_margin +threshold_scope: task diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..ad44bd37e51f82471f7a17d7372cee5fed00ed3f --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +evaluated learned fallback rule on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a387c5681dbf8c2cfdcff4979cee1ddd31d319 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,8964 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_model_selection": { + "base_success": 0.2777777777777778, + "base_utility": 0.7007981554626086, + "coverage": 0.6875, + "fallback_rate": 0.3125, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.5462962962962963, + "proposal_oracle_success": 0.5370370370370371, + "proposal_oracle_utility": 1.1440672337592803, + "selected_success": 0.39351851851851855, + "selected_success_gain_over_base": 0.11574074074074074, + "selected_utility": 0.882196049757674, + "selector_regret": 0.35351750337783555, + "success_selector_gap": 0.1875, + "success_support_gap": 0.17592592592592593, + "support_gap": 0.35465237537627453 + }, + "calibration_summary": { + "base_success": 0.2777777777777778, + "base_utility": 0.7007981554626086, + "coverage": 0.6875, + "fallback_rate": 0.3125, + "hidden_chart_oracle_success": 0.6597222222222222, + "hidden_chart_oracle_utility": 1.4150809271054134, + "outcome_ptr": 0.5462962962962963, + "pairwise_causal_calibration_accuracy": 0.5132708410132182, + "pairwise_causal_calibration_confidence": 0.530477852294317, + "pairwise_causal_calibration_ece": 0.01720701128109841, + "pairwise_causal_calibration_pairs": 37978.0, + "proposal_oracle_success": 0.5370370370370371, + "proposal_oracle_utility": 1.1440672337592803, + "selected_success": 0.39351851851851855, + "selected_success_gain_over_base": 0.11574074074074074, + "selected_utility": 0.882196049757674, + "selector_regret": 0.35351750337783555, + "success_selector_gap": 0.1875, + "success_support_gap": 0.17592592592592593, + "support_gap": 0.35465237537627453 + }, + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.7708333333333334, + "fallback_rate": 0.22916666666666666, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "pairwise_causal_calibration_accuracy": 0.5131896413060962, + "pairwise_causal_calibration_confidence": 0.5282932823510768, + "pairwise_causal_calibration_ece": 0.015103641044980439, + "pairwise_causal_calibration_pairs": 12434.0, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.2708333333333333, + "selected_success_gain_over_base": -0.020833333333333332, + "selected_utility": 0.6808734688885225, + "selector_regret": 0.5607294605369033, + "success_selector_gap": 0.3055555555555556, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_mean": [ + 0.0, + 4.602051421289207, + 1.8887871984592688, + -1.985099232970326e-17, + 7.187557603686636, + 0.0663594470046083, + 2.0903225806451613, + 0.7930022866760579, + 1.9819565879721794, + 15.375115207373272, + 2.788372394309843, + 5.701382488479263, + 12.81136712749616, + 0.40686157587079336, + 0.8936925909045862, + 0.24551906257091227, + 0.12993606097190302, + 0.1850963060060051, + 0.0663594470046083, + 3.944723778791356, + 4.364277845709234, + 4.603718512944113, + 5.60843294300425, + 4.854070660522273, + 0.3680969062939123, + 1.0, + 0.3217802000866517, + 4.613985697387828, + 0.5009397088659758 + ], + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "bundle_num_candidates_log", + "bundle_neighbor_count_r020", + "bundle_neighbor_count_r040", + "bundle_neighbor_frac_r020", + "bundle_neighbor_frac_r040", + "bundle_mean_peer_rms", + "bundle_min_peer_rms", + "bundle_medoid_rms", + "bundle_is_medoid", + "bundle_peer_score_mean_r020", + "bundle_peer_score_max_r020", + "bundle_peer_score_mean_r040", + "bundle_peer_score_max_r040", + "bundle_unique_source_count_r040", + "bundle_unique_source_frac_r040", + "bundle_unique_task_count_r040", + "bundle_same_source_frac_r040", + "bundle_score_density_r040", + "bundle_rank_density_r040" + ], + "feature_set": "bundle_consensus", + "feature_std": [ + 1.0, + 2.5232884736710055, + 2.2444575887945213, + 0.9999992484766103, + 4.542613580634851, + 0.24891036261589178, + 1.9642314545113713, + 0.37078681697003224, + 0.7171840796063713, + 1.7485561930736828, + 0.12987037363894735, + 4.878517299430629, + 3.217377946148024, + 0.346567593200359, + 0.19990387562321757, + 0.09641872137117102, + 0.07067998005028978, + 0.11663352509384882, + 0.2489103626158918, + 2.8138515595422326, + 2.965671838859836, + 2.4297066599961332, + 2.38657189782847, + 2.243933495797979, + 0.19723478821626303, + 1e-06, + 0.28108282108993726, + 2.4172277778814877, + 0.31354917609205285 + ], + "fit_design": { + "fit_objective": "pointwise", + "num_candidate_rows": 6510, + "num_fit_rows": 6510, + "num_pairwise_rows": 0, + "num_pointwise_rows": 6510, + "pairwise_weight": 0.0 + }, + "fit_objective": "pointwise", + "k": 16, + "num_calibration_candidates": 6510, + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "pairwise_causal_calibration": { + "calibration": { + "accuracy": 0.5132708410132182, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.012197836422904063, + "accuracy": 0.5159922200010806, + "confidence": 0.5281900564239846, + "count": 37018, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.21036288096926326, + "accuracy": 0.4083333333333333, + "confidence": 0.6186962143025966, + "count": 960, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.01720701128109841, + "mean_confidence": 0.530477852294317, + "n_bins": 10, + "num_pairs": 37978, + "num_rows": 432 + }, + "eval": { + "accuracy": 0.5131896413060962, + "bins": [ + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.0, + "upper": 0.1 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.1, + "upper": 0.2 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.2, + "upper": 0.3 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.3, + "upper": 0.4 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.4, + "upper": 0.5 + }, + { + "abs_gap": 0.012850915829321519, + "accuracy": 0.5137396439996719, + "confidence": 0.5265905598289934, + "count": 12191, + "lower": 0.5, + "upper": 0.6 + }, + { + "abs_gap": 0.12811999126760554, + "accuracy": 0.48559670781893005, + "confidence": 0.6137166990865356, + "count": 243, + "lower": 0.6, + "upper": 0.7 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.7, + "upper": 0.8 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.8, + "upper": 0.9 + }, + { + "abs_gap": 0.0, + "accuracy": 0.0, + "confidence": 0.0, + "count": 0, + "lower": 0.9, + "upper": 1.0 + } + ], + "ece": 0.015103641044980439, + "mean_confidence": 0.5282932823510768, + "n_bins": 10, + "num_pairs": 12434, + "num_rows": 144 + } + }, + "pairwise_weight": 1.0, + "report_type": "learned_dominance_selector_eval", + "ridge_lambdas": [ + 0.0, + 0.01, + 0.1, + 1.0, + 10.0, + 100.0 + ], + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3025210084033613, + "pairwise_causal_calibration_confidence": 0.5305966527751617, + "pairwise_causal_calibration_ece": 0.22807564437180022, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.2180007600904328, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7026570439338684, + "selected_utility_gain_over_base": -0.020948290824890137, + "selector_regret": 1.2973429560661316, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5343458881100447, + "pairwise_causal_calibration_ece": 0.10577445953861614, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.3084681084329295, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3865546218487395, + "pairwise_causal_calibration_confidence": 0.5251040975256128, + "pairwise_causal_calibration_ece": 0.13854947567687334, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.2551750656075202, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.67014479637146, + "selected_utility_gain_over_base": -1.32985520362854, + "selector_regret": 1.32985520362854, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5904761904761905, + "pairwise_causal_calibration_confidence": 0.5293645053307068, + "pairwise_causal_calibration_ece": 0.14243134895449866, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.08072228079745068, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5381289217490534, + "pairwise_causal_calibration_ece": 0.15631073993087152, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.009282072163031563, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20389758050441742, + "selected_utility_gain_over_base": -0.49605758488178253, + "selector_regret": 1.7961024194955826, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6944444444444444, + "pairwise_causal_calibration_confidence": 0.5457014445746324, + "pairwise_causal_calibration_ece": 0.14874299986981188, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.23967836808523524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.02896384295113541, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.3567590128192366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4188034188034188, + "pairwise_causal_calibration_confidence": 0.5224444135096841, + "pairwise_causal_calibration_ece": 0.10364099470626525, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.07828613505539983, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49166666666666664, + "pairwise_causal_calibration_confidence": 0.5251639529546277, + "pairwise_causal_calibration_ece": 0.03349728628796106, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4675581452853047, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02898651361465454, + "selected_utility_gain_over_base": -0.06491655111312866, + "selector_regret": 0.03121185302734375, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6916666666666667, + "pairwise_causal_calibration_confidence": 0.5293656296599286, + "pairwise_causal_calibration_ece": 0.16230103700673804, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4639337683699436, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.010183374397456646, + "selected_utility_gain_over_base": 0.008253872743807733, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45, + "pairwise_causal_calibration_confidence": 0.5138396482606322, + "pairwise_causal_calibration_ece": 0.06383964826063221, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.11708670033383684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07831952720880508, + "selected_utility_gain_over_base": -1.921680472791195, + "selector_regret": 0.07624677568674088, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5294117647058824, + "pairwise_causal_calibration_confidence": 0.5321127842961938, + "pairwise_causal_calibration_ece": 0.0027010195903114376, + "pairwise_causal_calibration_pairs": 17.0, + "predicted_margin": -0.11406027153604881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.521287134073598, + "pairwise_causal_calibration_ece": 0.028712865926402098, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.08424855703727296, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08540759235620499, + "selected_utility_gain_over_base": 0.00841909646987915, + "selector_regret": 0.0651073232293129, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5166835288855317, + "pairwise_causal_calibration_ece": 0.10001686221886502, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.029164940771620018, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05741874501109123, + "selected_utility_gain_over_base": 0.021532438695430756, + "selector_regret": 0.08929390087723732, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2608695652173913, + "pairwise_causal_calibration_confidence": 0.5171237518534777, + "pairwise_causal_calibration_ece": 0.2562541866360864, + "pairwise_causal_calibration_pairs": 46.0, + "predicted_margin": -0.13774783214464598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5253895829839477, + "pairwise_causal_calibration_ece": 0.007943750349385659, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2314722991491765, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.012751738540828228, + "selected_utility_gain_over_base": 0.004484554752707481, + "selector_regret": 0.0664558382704854, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6333333333333333, + "pairwise_causal_calibration_confidence": 0.5217604331896001, + "pairwise_causal_calibration_ece": 0.11157290014373322, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.2496492910982782, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44047619047619047, + "pairwise_causal_calibration_confidence": 0.5212774691287471, + "pairwise_causal_calibration_ece": 0.0808012786525566, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.08651310891179698, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4583333333333333, + "pairwise_causal_calibration_confidence": 0.5279271325774133, + "pairwise_causal_calibration_ece": 0.06959379924407999, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.008737131332534287, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007268309593200684, + "selected_utility_gain_over_base": -0.09556593745946884, + "selector_regret": 0.007909595966339111, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.5476280294891419, + "pairwise_causal_calibration_ece": 0.03929469615580864, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.026393440334875118, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11457405239343643, + "selected_utility_gain_over_base": -0.02813490480184555, + "selector_regret": 0.10201253741979599, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5395580589892535, + "pairwise_causal_calibration_ece": 0.006224725655920161, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8489010665175709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08826130628585815, + "selected_utility_gain_over_base": 0.005126260221004486, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5209300433514414, + "pairwise_causal_calibration_ece": 0.08759671001810809, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.02787377016220005, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019201835617423058, + "selected_utility_gain_over_base": -0.034921107813715935, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49333333333333335, + "pairwise_causal_calibration_confidence": 0.5169520118772317, + "pairwise_causal_calibration_ece": 0.023618678543898375, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": -0.07560538396524885, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5768622164838934, + "pairwise_causal_calibration_ece": 0.5768622164838934, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.08574322718445004, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04285776615142822, + "selected_utility_gain_over_base": -1.9571422338485718, + "selector_regret": 1.9571422338485718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6333333333333333, + "pairwise_causal_calibration_confidence": 0.5402794532595608, + "pairwise_causal_calibration_ece": 0.09305388007377255, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5479017224682395, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015753567218780518, + "selected_utility_gain_over_base": -0.07724392414093018, + "selector_regret": 0.025640826672315598, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.32844237707236035, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38333333333333336, + "pairwise_causal_calibration_confidence": 0.522326045653055, + "pairwise_causal_calibration_ece": 0.1389927123197216, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.04417027513292293, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04841095209121704, + "selected_utility_gain_over_base": -1.951589047908783, + "selector_regret": 0.059984683990478516, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.696969696969697, + "pairwise_causal_calibration_confidence": 0.5229830501890643, + "pairwise_causal_calibration_ece": 0.1739866467806327, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.1757789642733626, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07767494767904282, + "selected_utility_gain_over_base": 0.028091274201869965, + "selector_regret": 1.9223250523209572, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5337482570523958, + "pairwise_causal_calibration_ece": 0.21625174294760421, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5713459023104054, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03462250903248787, + "selected_utility_gain_over_base": -0.06989232823252678, + "selector_regret": 0.026248913258314133, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.525, + "pairwise_causal_calibration_confidence": 0.539953848754645, + "pairwise_causal_calibration_ece": 0.15111259204903227, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.05440518485836532, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9280846938490868, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6190476190476191, + "pairwise_causal_calibration_confidence": 0.5223339125255857, + "pairwise_causal_calibration_ece": 0.09671370652203337, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.7190808236152633, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5862068965517241, + "pairwise_causal_calibration_confidence": 0.5272168394784098, + "pairwise_causal_calibration_ece": 0.058990057073314284, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.7070441087351717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5267509882327646, + "pairwise_causal_calibration_ece": 0.043413085733646106, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6155609171443972, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.054066285490989685, + "selected_utility_gain_over_base": -0.052404239773750305, + "selector_regret": 0.08999375998973846, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6272727272727273, + "pairwise_causal_calibration_confidence": 0.5316179775586928, + "pairwise_causal_calibration_ece": 0.0956547497140344, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.5329093436927219, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5050505050505051, + "pairwise_causal_calibration_confidence": 0.5386347584075308, + "pairwise_causal_calibration_ece": 0.03358425335702575, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.3447490352381858, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5378151260504201, + "pairwise_causal_calibration_confidence": 0.5162619682864167, + "pairwise_causal_calibration_ece": 0.02155315776400346, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.5136988024654346, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.41414141414141414, + "pairwise_causal_calibration_confidence": 0.5226475437853878, + "pairwise_causal_calibration_ece": 0.10850612964397366, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.6476723025740536, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07608599960803986, + "selected_utility_gain_over_base": -0.010948076844215393, + "selector_regret": 1.9239140003919601, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6388888888888888, + "pairwise_causal_calibration_confidence": 0.5184746997192393, + "pairwise_causal_calibration_ece": 0.12041418916964952, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.13802527783024798, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.043751150369644165, + "selected_utility_gain_over_base": -0.008499354124069214, + "selector_regret": 0.054248154163360596, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6388888888888888, + "pairwise_causal_calibration_confidence": 0.5310489483070069, + "pairwise_causal_calibration_ece": 0.10783994058188195, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.09399166521150604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12222731113433838, + "selected_utility_gain_over_base": -0.040779367089271545, + "selector_regret": 0.015101373195648193, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5379810554394029, + "pairwise_causal_calibration_ece": 0.06571145427295252, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15799813692233977, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09047546982765198, + "selected_utility_gain_over_base": -0.06794795393943787, + "selector_regret": 0.012273147702217102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5470085470085471, + "pairwise_causal_calibration_confidence": 0.5361003534463802, + "pairwise_causal_calibration_ece": 0.042038781082623024, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.11714129730488587, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14173981547355652, + "selected_utility_gain_over_base": -0.6823966801166534, + "selector_regret": 1.8582601845264435, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5158281275240727, + "pairwise_causal_calibration_ece": 0.08249479419073935, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.12116588314285559, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7058823529411765, + "pairwise_causal_calibration_confidence": 0.5351161430732934, + "pairwise_causal_calibration_ece": 0.17076620986788324, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.16494316577956913, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12586313486099243, + "selected_utility_gain_over_base": -0.06504528224468231, + "selector_regret": 1.8741368651390076, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4095238095238095, + "pairwise_causal_calibration_confidence": 0.51710870319415, + "pairwise_causal_calibration_ece": 0.10758489367034052, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.030788448510796762, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15656626224517822, + "selected_utility_gain_over_base": -0.06943503022193909, + "selector_regret": 0.7097858786582947, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5333367283214293, + "pairwise_causal_calibration_ece": 0.14047958546428646, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1742924240977557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.028782829642295837, + "selected_utility_gain_over_base": -0.06342208385467529, + "selector_regret": 0.026206374168395996, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5385441284405091, + "pairwise_causal_calibration_ece": 0.06175922923362397, + "pairwise_causal_calibration_pairs": 42.0, + "predicted_margin": -0.0742733989521003, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35, + "pairwise_causal_calibration_confidence": 0.5258131375323639, + "pairwise_causal_calibration_ece": 0.17581313753236394, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": -0.10215255174920451, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14946338534355164, + "selected_utility_gain_over_base": -1.8505366146564484, + "selector_regret": 1.8505366146564484, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2571428571428571, + "pairwise_causal_calibration_confidence": 0.5267116008364124, + "pairwise_causal_calibration_ece": 0.2695687436935554, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": -0.11408015116078918, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14746195077896118, + "selected_utility_gain_over_base": -1.8525380492210388, + "selector_regret": 1.8525380492210388, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5238503061013462, + "pairwise_causal_calibration_ece": 0.09885030610134621, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.16726605949103523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6900332570075989, + "selected_utility_gain_over_base": -0.03357207775115967, + "selector_regret": 1.3099667429924011, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.46153846153846156, + "pairwise_causal_calibration_confidence": 0.5333254751964351, + "pairwise_causal_calibration_ece": 0.07178701365797358, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.22321666120891376, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3416666666666667, + "pairwise_causal_calibration_confidence": 0.5423238840429205, + "pairwise_causal_calibration_ece": 0.20065721737625392, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.20299343323495855, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6947963237762451, + "selected_utility_gain_over_base": -1.3052036762237549, + "selector_regret": 1.3052036762237549, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5357192536556177, + "pairwise_causal_calibration_ece": 0.0682003715539854, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.1274552507433626, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.509090909090909, + "pairwise_causal_calibration_confidence": 0.5333064538046144, + "pairwise_causal_calibration_ece": 0.02421554471370535, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.11963786997800063, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5111398905599821, + "pairwise_causal_calibration_ece": 0.04441566499557348, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.2514915262180857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.02896384295113541, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.19263300195342614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4117647058823529, + "pairwise_causal_calibration_confidence": 0.5398627136285538, + "pairwise_causal_calibration_ece": 0.12809800774620084, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.17661137540295557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5166360139443322, + "pairwise_causal_calibration_ece": 0.08330268061099888, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4500223738950829, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009051382541656494, + "selected_utility_gain_over_base": -0.08485168218612671, + "selector_regret": 0.007533630356192589, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44166666666666665, + "pairwise_causal_calibration_confidence": 0.5157561573212285, + "pairwise_causal_calibration_ece": 0.07408949065456183, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.10658218471121564, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00289076566696167, + "selected_utility_gain_over_base": 0.000961264013312757, + "selector_regret": 0.0070051951333880424, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49122807017543857, + "pairwise_causal_calibration_confidence": 0.5227936475539869, + "pairwise_causal_calibration_ece": 0.03156557737854837, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.008532620847146344, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13208407163619995, + "selected_utility_gain_over_base": -1.8679159283638, + "selector_regret": 1.8679159283638, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5540392504779817, + "pairwise_causal_calibration_ece": 0.06500635112191766, + "pairwise_causal_calibration_pairs": 35.0, + "predicted_margin": -0.16708550898936614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.5281702802889924, + "pairwise_causal_calibration_ece": 0.0531702802889924, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.034705384024887875, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11253257840871811, + "selected_utility_gain_over_base": 0.03554408252239227, + "selector_regret": 0.09649785608053207, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.5226825874028909, + "pairwise_causal_calibration_ece": 0.047682587402890886, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1403583050387357, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01919553615152836, + "selected_utility_gain_over_base": -0.016690770164132118, + "selector_regret": 0.05615365318953991, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38461538461538464, + "pairwise_causal_calibration_confidence": 0.5121094164593272, + "pairwise_causal_calibration_ece": 0.12749403184394253, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.07780183911371158, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.525, + "pairwise_causal_calibration_confidence": 0.5270537692784654, + "pairwise_causal_calibration_ece": 0.002053769278465345, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3835862173881076, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.028474071994423866, + "selected_utility_gain_over_base": 0.02020688820630312, + "selector_regret": 0.03328571654856205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49572649572649574, + "pairwise_causal_calibration_confidence": 0.5204348846022294, + "pairwise_causal_calibration_ece": 0.024708388875733678, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.03727831765723297, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0021741788368672132, + "selected_utility_gain_over_base": -1.9978258211631328, + "selector_regret": 1.9978258211631328, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4533333333333333, + "pairwise_causal_calibration_confidence": 0.5302663446880443, + "pairwise_causal_calibration_ece": 0.076933011354711, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.09881997958173372, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5244535198032577, + "pairwise_causal_calibration_ece": 0.05054648019674224, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.11792927817531362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030119141563773155, + "selected_utility_gain_over_base": -0.07271510548889637, + "selector_regret": 0.006891986355185509, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5833333333333334, + "pairwise_causal_calibration_confidence": 0.5331520837039857, + "pairwise_causal_calibration_ece": 0.05395327484480136, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.23958299592081822, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5207010539912516, + "pairwise_causal_calibration_ece": 0.029298946008748494, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.39916002861669686, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017789265140891075, + "selected_utility_gain_over_base": -0.0653457809239626, + "selector_regret": 0.04962191917002201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.375, + "pairwise_causal_calibration_confidence": 0.528181691198459, + "pairwise_causal_calibration_ece": 0.15318169119845904, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.1626450174749542, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006599446292966604, + "selected_utility_gain_over_base": -0.04752349713817239, + "selector_regret": 0.03618248412385583, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5789473684210527, + "pairwise_causal_calibration_confidence": 0.5298121533009607, + "pairwise_causal_calibration_ece": 0.049135215120092, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.4256345336980503, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5740740740740741, + "pairwise_causal_calibration_confidence": 0.5594515320080404, + "pairwise_causal_calibration_ece": 0.2553362522454674, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.1354107312118617, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3416666666666667, + "pairwise_causal_calibration_confidence": 0.5288603912492978, + "pairwise_causal_calibration_ece": 0.18719372458263117, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.30245869311688867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06182505562901497, + "selected_utility_gain_over_base": -0.031172435730695724, + "selector_regret": 0.04968476668000221, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.5627089760709729, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.5240130464861401, + "pairwise_causal_calibration_ece": 0.04901304648614013, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.15678779044109292, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7065217391304348, + "pairwise_causal_calibration_confidence": 0.5170017754722115, + "pairwise_causal_calibration_ece": 0.18951996365822332, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 0.18741646794311717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.375, + "pairwise_causal_calibration_confidence": 0.5262268456146021, + "pairwise_causal_calibration_ece": 0.1512268456146021, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3107421704514528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5415632384984795, + "pairwise_causal_calibration_ece": 0.10876536091501116, + "pairwise_causal_calibration_pairs": 49.0, + "predicted_margin": -0.03958497742915945, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9280846938490868, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35384615384615387, + "pairwise_causal_calibration_confidence": 0.5300108478221505, + "pairwise_causal_calibration_ece": 0.1761646939759966, + "pairwise_causal_calibration_pairs": 65.0, + "predicted_margin": -0.670987032962785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8666666666666667, + "pairwise_causal_calibration_confidence": 0.5379425484601451, + "pairwise_causal_calibration_ece": 0.3287241182065216, + "pairwise_causal_calibration_pairs": 15.0, + "predicted_margin": 0.7017571830152312, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 14, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.5232561219266944, + "pairwise_causal_calibration_ece": 0.035861163943501084, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.24134426910254947, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.543859649122807, + "pairwise_causal_calibration_confidence": 0.5328964743528634, + "pairwise_causal_calibration_ece": 0.010963174769943684, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 0.5122492091762928, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.5422560685706671, + "pairwise_causal_calibration_ece": 0.048653022338423835, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.2330187222265443, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5280283691474993, + "pairwise_causal_calibration_ece": 0.04340020228107211, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.4261006779620051, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.5136404664974943, + "pairwise_causal_calibration_ece": 0.02624550851430102, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.38625575833343895, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19651852548122406, + "selected_utility_gain_over_base": 0.10948444902896881, + "selector_regret": 1.803481474518776, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4722222222222222, + "pairwise_causal_calibration_confidence": 0.5119131115242276, + "pairwise_causal_calibration_ece": 0.03969088930200537, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.0925196999812683, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07580336928367615, + "selected_utility_gain_over_base": 0.02355286478996277, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5070899546645378, + "pairwise_causal_calibration_ece": 0.06264551022009335, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.07407655216726083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12105989456176758, + "selected_utility_gain_over_base": -0.041946783661842346, + "selector_regret": 0.6918159127235413, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5252684472642476, + "pairwise_causal_calibration_ece": 0.1890172670214667, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13463328766802296, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18093222379684448, + "selected_utility_gain_over_base": 0.02250880002975464, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5454545454545454, + "pairwise_causal_calibration_confidence": 0.5394992318577098, + "pairwise_causal_calibration_ece": 0.038926080840327754, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": -0.09068349284851132, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.30158730158730157, + "pairwise_causal_calibration_confidence": 0.5242300284642385, + "pairwise_causal_calibration_ece": 0.2226427268769369, + "pairwise_causal_calibration_pairs": 63.0, + "predicted_margin": -0.10941040668136354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8129013180732727, + "selected_utility_gain_over_base": -0.02050870656967163, + "selector_regret": 1.1870986819267273, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8416666666666667, + "pairwise_causal_calibration_confidence": 0.5382974229033135, + "pairwise_causal_calibration_ece": 0.30336924376335306, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.10202293411527177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.580952380952381, + "pairwise_causal_calibration_confidence": 0.5316022421578545, + "pairwise_causal_calibration_ece": 0.049350138794526544, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.05730648283766754, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8148510456085205, + "selected_utility_gain_over_base": 0.5888497531414032, + "selector_regret": 1.1851489543914795, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5357950970847783, + "pairwise_causal_calibration_ece": 0.1786522399419212, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18792055489105874, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06454251706600189, + "selected_utility_gain_over_base": -0.02766239643096924, + "selector_regret": 0.067916139960289, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5308940956422026, + "pairwise_causal_calibration_ece": 0.1357725710244641, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.061055849565622045, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38461538461538464, + "pairwise_causal_calibration_confidence": 0.523423235146417, + "pairwise_causal_calibration_ece": 0.13880785053103237, + "pairwise_causal_calibration_pairs": 39.0, + "predicted_margin": -0.15959785934997645, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125194311141968, + "selected_utility_gain_over_base": -1.1874805688858032, + "selector_regret": 1.1874805688858032, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44285714285714284, + "pairwise_causal_calibration_confidence": 0.5323449915161098, + "pairwise_causal_calibration_ece": 0.10072366188957278, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": -0.17466576670619666, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5242257655342142, + "pairwise_causal_calibration_ece": 0.024225765534214183, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 0.24853085052742357, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6829424500465393, + "selected_utility_gain_over_base": -0.04066288471221924, + "selector_regret": 1.3170575499534607, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3626373626373626, + "pairwise_causal_calibration_confidence": 0.5472048889164063, + "pairwise_causal_calibration_ece": 0.1845675262790436, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.04845957189152705, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20841218531131744, + "selected_utility_gain_over_base": -0.3916783779859543, + "selector_regret": 0.021801263093948364, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8809523809523809, + "pairwise_causal_calibration_confidence": 0.5368743958607693, + "pairwise_causal_calibration_ece": 0.3440779850916114, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.24613098923868526, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44166666666666665, + "pairwise_causal_calibration_confidence": 0.5315196569218078, + "pairwise_causal_calibration_ece": 0.08985299025514107, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.030776181488394214, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21245408058166504, + "selected_utility_gain_over_base": -0.513247549533844, + "selector_regret": 0.49927353858947754, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5636363636363636, + "pairwise_causal_calibration_confidence": 0.5375543539918383, + "pairwise_causal_calibration_ece": 0.03382332741347877, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.08224411989102232, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": -0.07385947182024077, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3611111111111111, + "pairwise_causal_calibration_confidence": 0.525252740289532, + "pairwise_causal_calibration_ece": 0.16414162917842084, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.28676975669762994, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.02896384295113541, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.2551836974507351, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.5251781957501036, + "pairwise_causal_calibration_ece": 0.03778323776691034, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.3518959688723307, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.519839782973733, + "pairwise_causal_calibration_ece": 0.08016021702626697, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4462633698719307, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06987287849187851, + "selected_utility_gain_over_base": -0.024030186235904694, + "selector_regret": 0.03010360151529312, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48333333333333334, + "pairwise_causal_calibration_confidence": 0.5355453827080833, + "pairwise_causal_calibration_ece": 0.052212049374749936, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2233295555290654, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00974698830395937, + "selected_utility_gain_over_base": 0.007817486650310457, + "selector_regret": 0.029104947112500668, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4369747899159664, + "pairwise_causal_calibration_confidence": 0.5252379082880491, + "pairwise_causal_calibration_ece": 0.08826311837208267, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.043465415221497966, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05522008612751961, + "selected_utility_gain_over_base": -1.9447799138724804, + "selector_regret": 1.9447799138724804, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.13333333333333333, + "pairwise_causal_calibration_confidence": 0.5470577940231715, + "pairwise_causal_calibration_ece": 0.41372446068983815, + "pairwise_causal_calibration_pairs": 30.0, + "predicted_margin": -0.1772708238485603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6166666666666667, + "pairwise_causal_calibration_confidence": 0.5196927968287194, + "pairwise_causal_calibration_ece": 0.09697386983794731, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.03512423535406803, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21886132657527924, + "selected_utility_gain_over_base": 0.1418728306889534, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.45, + "pairwise_causal_calibration_confidence": 0.5144356104774866, + "pairwise_causal_calibration_ece": 0.06443561047748664, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.0006652866876127872, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0682595744729042, + "selected_utility_gain_over_base": 0.03237326815724373, + "selector_regret": 0.2036382481455803, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.52, + "pairwise_causal_calibration_confidence": 0.5170849899671911, + "pairwise_causal_calibration_ece": 0.002915010032808918, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.12444815357019567, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4666666666666667, + "pairwise_causal_calibration_confidence": 0.5281218746338447, + "pairwise_causal_calibration_ece": 0.06145520796717807, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2276785617416764, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05094657465815544, + "selected_utility_gain_over_base": 0.042679390870034695, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6239316239316239, + "pairwise_causal_calibration_confidence": 0.5230485750660122, + "pairwise_causal_calibration_ece": 0.10088304886561172, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.12368708414396264, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 12, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.522915616847175, + "pairwise_causal_calibration_ece": 0.15890256497100674, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.0589756379475741, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08930286020040512, + "selected_utility_gain_over_base": -0.09241268783807755, + "selector_regret": 1.9106971397995949, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.538380386098592, + "pairwise_causal_calibration_ece": 0.043205574824282285, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.034121695817442343, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.058951716870069504, + "selected_utility_gain_over_base": -0.04388253018260002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.725, + "pairwise_causal_calibration_confidence": 0.546674773294179, + "pairwise_causal_calibration_ece": 0.1783252267058209, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.043765176280211815, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1975085586309433, + "selected_utility_gain_over_base": 0.054799601435661316, + "selector_regret": 1.8024914413690567, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5583333333333333, + "pairwise_causal_calibration_confidence": 0.5159516842599035, + "pairwise_causal_calibration_ece": 0.042381649073429895, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.47473165550706736, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03889387845993042, + "selected_utility_gain_over_base": -0.04424116760492325, + "selector_regret": 0.03675774484872818, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6166666666666667, + "pairwise_causal_calibration_confidence": 0.523340132602306, + "pairwise_causal_calibration_ece": 0.09332653406436064, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.22174183764619265, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.028320351615548134, + "selected_utility_gain_over_base": -0.02580259181559086, + "selector_regret": 0.02148670144379139, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44047619047619047, + "pairwise_causal_calibration_confidence": 0.5313546391394341, + "pairwise_causal_calibration_ece": 0.09087844866324368, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.3658750831677004, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5151515151515151, + "pairwise_causal_calibration_confidence": 0.5363991279304782, + "pairwise_causal_calibration_ece": 0.03713806757575985, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.08745956040002545, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12977690994739532, + "selected_utility_gain_over_base": -1.8702230900526047, + "selector_regret": 1.8702230900526047, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35833333333333334, + "pairwise_causal_calibration_confidence": 0.5188748176574249, + "pairwise_causal_calibration_ece": 0.1605414843240916, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.26743135206183577, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03218920901417732, + "selected_utility_gain_over_base": -0.06080828234553337, + "selector_regret": 0.019156794995069504, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": NaN, + "pairwise_causal_calibration_confidence": NaN, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.2217357274409076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45, + "pairwise_causal_calibration_confidence": 0.5277135352836209, + "pairwise_causal_calibration_ece": 0.07771353528362085, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.08370080144449699, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06004788354039192, + "selected_utility_gain_over_base": -1.939952116459608, + "selector_regret": 0.06505399569869041, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.2222222222222222, + "pairwise_causal_calibration_confidence": 0.5366211565538932, + "pairwise_causal_calibration_ece": 0.31439893433167077, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.33781119137638194, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25833333333333336, + "pairwise_causal_calibration_confidence": 0.528533525486533, + "pairwise_causal_calibration_ece": 0.2702001921531997, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6769572594134817, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04690708592534065, + "selected_utility_gain_over_base": -0.057607751339673996, + "selector_regret": 0.06619475409388542, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5237701807590434, + "pairwise_causal_calibration_ece": 0.026229819240956687, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.16858629361831445, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6190476190476191, + "pairwise_causal_calibration_confidence": 0.5351820044263471, + "pairwise_causal_calibration_ece": 0.08386561462127182, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.5511814245803944, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": -0.0718611703384171, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7222222222222222, + "pairwise_causal_calibration_confidence": 0.536464978436527, + "pairwise_causal_calibration_ece": 0.18575724378569516, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.6922087959324436, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6181818181818182, + "pairwise_causal_calibration_confidence": 0.5209245623930986, + "pairwise_causal_calibration_ece": 0.09725725578871958, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": -0.06739236477010135, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43636363636363634, + "pairwise_causal_calibration_confidence": 0.5352532151194661, + "pairwise_causal_calibration_ece": 0.0988895787558297, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.4672762110936981, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13743820786476135, + "selected_utility_gain_over_base": 0.012670740485191345, + "selector_regret": 1.8625617921352386, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5952380952380952, + "pairwise_causal_calibration_confidence": 0.5426310160661976, + "pairwise_causal_calibration_ece": 0.05260707917189755, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.3117657280355065, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08501242101192474, + "selected_utility_gain_over_base": -1.9149875789880753, + "selector_regret": 1.9149875789880753, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6302521008403361, + "pairwise_causal_calibration_confidence": 0.5305902079798236, + "pairwise_causal_calibration_ece": 0.09966189286051252, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.3587989801510558, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 5.424022674560547e-06, + "selected_utility_gain_over_base": 5.200505256652832e-06, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5882352941176471, + "pairwise_causal_calibration_confidence": 0.5148540856771467, + "pairwise_causal_calibration_ece": 0.07338120844050033, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.35255742400707646, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21615806221961975, + "selected_utility_gain_over_base": 0.1291239857673645, + "selector_regret": 1.7838419377803802, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.28744401244938833, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8055555555555556, + "pairwise_causal_calibration_confidence": 0.5138245455754116, + "pairwise_causal_calibration_ece": 0.291731009980144, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.061796487673025036, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11105132102966309, + "selected_utility_gain_over_base": 0.05880081653594971, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5234811460445998, + "pairwise_causal_calibration_ece": 0.10681447937793315, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.07294104079981856, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21447330713272095, + "selected_utility_gain_over_base": 0.05146662890911102, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5332614138108104, + "pairwise_causal_calibration_ece": 0.0024528719034753177, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0627076273903946, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14433109760284424, + "selected_utility_gain_over_base": -0.014092326164245605, + "selector_regret": 0.05094057321548462, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3949579831932773, + "pairwise_causal_calibration_confidence": 0.5340698575932721, + "pairwise_causal_calibration_ece": 0.1391118743999947, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.14154551725382616, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1460946798324585, + "selected_utility_gain_over_base": -0.6780418157577515, + "selector_regret": 1.8539053201675415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5111111111111111, + "pairwise_causal_calibration_confidence": 0.5240930381172986, + "pairwise_causal_calibration_ece": 0.012981927006187598, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.14659151504624657, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16154873371124268, + "selected_utility_gain_over_base": -0.6718612909317017, + "selector_regret": 1.8384512662887573, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5317298952114761, + "pairwise_causal_calibration_ece": 0.06827010478852391, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.26370842901349006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04297435283660889, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4666666666666667, + "pairwise_causal_calibration_confidence": 0.5177275439295085, + "pairwise_causal_calibration_ece": 0.05106087726284181, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.033176433226523416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18784314393997192, + "selected_utility_gain_over_base": -0.038158148527145386, + "selector_regret": 0.36436158418655396, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5266268320102542, + "pairwise_causal_calibration_ece": 0.026626832010254176, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1739828724101382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13575458526611328, + "selected_utility_gain_over_base": 0.04354967176914215, + "selector_regret": 0.05715706944465637, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8888888888888888, + "pairwise_causal_calibration_confidence": 0.5566493339472486, + "pairwise_causal_calibration_ece": 0.3322395549416404, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": -0.00871862718880162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6309523809523809, + "pairwise_causal_calibration_confidence": 0.5248002449882088, + "pairwise_causal_calibration_ece": 0.12044288641048877, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.20549642192166445, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5657894736842105, + "pairwise_causal_calibration_confidence": 0.5344673895535903, + "pairwise_causal_calibration_ece": 0.031322084130620015, + "pairwise_causal_calibration_pairs": 76.0, + "predicted_margin": -0.16215465004219584, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8302689790725708, + "selected_utility_gain_over_base": -1.1697310209274292, + "selector_regret": 1.1697310209274292, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": -0.18710182505305797, + "train_seed": "2" + } + ], + "schema_version": 1, + "selected_lambda": 0.0, + "selector_calibration_target_content_hash": null, + "selector_calibration_target_split_hash": null, + "selector_chart_feature_mode": null, + "selector_eval_target_content_hash": null, + "selector_eval_target_split_hash": null, + "selector_source_content_hash": null, + "selector_source_split_hash": null, + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "success_bonus": 1.0, + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22916666666666666, + "mean": 0.30625, + "n": 120, + "num_samples": 100 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2864242424242424, + "low": 0.09454545454545454, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 100 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3541666666666667, + "low": 0.20833333333333334, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9292563527069028, + "low": 0.6413994663580929, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 100 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.880235356376845, + "low": 0.5457555404169147, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 100 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8549258244666917, + "low": 0.5884616619085945, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8444444444444444, + "low": 0.6986111111111112, + "mean": 0.7784722222222222, + "n": 120, + "num_samples": 100 + }, + "mean": 0.7784722222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.75, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8557575757575758, + "low": 0.32121212121212117, + "mean": 0.6345454545454545, + "n": 5, + "num_samples": 100 + }, + "mean": 0.6345454545454545, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.7333333333333333, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.9393939393939394 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.8333333333333334, + "low": 0.6944444444444444, + "mean": 0.7708333333333334, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8444444444444444, + "low": 0.6986111111111112, + "mean": 0.7784722222222222, + "n": 120, + "num_samples": 100 + }, + "mean": 0.7784722222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.75, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8557575757575758, + "low": 0.32121212121212117, + "mean": 0.6345454545454545, + "n": 5, + "num_samples": 100 + }, + "mean": 0.6345454545454545, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.7333333333333333, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.9393939393939394 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.8333333333333334, + "low": 0.6944444444444444, + "mean": 0.7708333333333334, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30138888888888893, + "low": 0.15555555555555556, + "mean": 0.22152777777777777, + "n": 120, + "num_samples": 100 + }, + "mean": 0.22152777777777777, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6787878787878788, + "low": 0.14424242424242423, + "mean": 0.3654545454545454, + "n": 5, + "num_samples": 100 + }, + "mean": 0.3654545454545454, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.26666666666666666, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.06060606060606061 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.3055555555555556, + "low": 0.16666666666666666, + "mean": 0.22916666666666666, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.85, + "low": 0.7277777777777777, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 100 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.712, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 100 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7916666666666666, + "low": 0.6666666666666666, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7339281444541284, + "low": 1.519337441979183, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 100 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.488072731614113, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 100 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.6290036176020901, + "low": 1.404496934161418, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6180555555555555, + "low": 0.44305555555555554, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 100 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7133333333333334, + "low": 0.23393939393939395, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 100 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4791666666666667, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "pairwise_causal_calibration_accuracy": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5364782188006679, + "low": 0.4879921023444142, + "mean": 0.511616029131761, + "n": 117, + "num_samples": 100 + }, + "mean": 0.511616029131761, + "per_seed": { + "1073161778/train0": 0.3865546218487395, + "1073161778/train1": 0.3416666666666667, + "1073161778/train2": 0.8809523809523809, + "1113371261/train0": 0.5, + "1113371261/train1": 0.7142857142857143, + "1113371261/train2": 0.5357142857142857, + "1151410346/train0": 0.4095238095238095, + "1151410346/train1": 0.580952380952381, + "1151410346/train2": 0.4666666666666667, + "1261930456/train0": 0.5050505050505051, + "1261930456/train1": 0.5909090909090909, + "1261930456/train2": 0.5952380952380952, + "1324133122/train0": 0.39285714285714285, + "1324133122/train1": 0.35714285714285715, + "1324133122/train2": 0.5, + "1324642411/train0": 0.55, + "1324642411/train1": 0.48739495798319327, + "1324642411/train2": 0.6181818181818182, + "1375492786/train0": 0.49333333333333335, + "1375492786/train1": 0.5789473684210527, + "1375492786/train2": 0.44047619047619047, + "141984615/train0": 0.4583333333333333, + "141984615/train1": 0.575, + "141984615/train2": 0.5083333333333333, + "1461069713/train0": 0.49166666666666664, + "1461069713/train1": 0.43333333333333335, + "1461069713/train2": 0.6, + "1480856810/train0": 0.41414141414141414, + "1480856810/train1": 0.48739495798319327, + "1480856810/train2": 0.5882352941176471, + "1509980664/train0": 0.43333333333333335, + "1509980664/train1": 0.30158730158730157, + "1509980664/train2": 0.5111111111111111, + "1522272686/train0": 0.6944444444444444, + "1522272686/train1": 0.5555555555555556, + "1522272686/train2": 0.3611111111111111, + "1549324805/train0": 0.75, + "1549324805/train1": 0.375, + "1549324805/train2": 0.25833333333333336, + "1578990937/train0": 0.5470085470085471, + "1578990937/train1": 0.5454545454545454, + "1578990937/train2": 0.3949579831932773, + "1790188787/train0": 0.6272727272727273, + "1790188787/train1": 0.543859649122807, + "1790188787/train2": 0.43636363636363634, + "1822210471/train0": 0.525, + "1822210471/train1": 0.5714285714285714, + "1822210471/train2": 0.55, + "1845489013/train0": 0.2608695652173913, + "1845489013/train1": 0.38461538461538464, + "1845489013/train2": 0.52, + "1999022574/train0": 0.2571428571428571, + "1999022574/train1": 0.44285714285714284, + "1999022574/train2": 0.5657894736842105, + "2012211287/train0": 0.3025210084033613, + "2012211287/train1": 0.425, + "2012211287/train2": 0.5, + "2038594191/train0": 0.0, + "2038594191/train1": 0.5740740740740741, + "2038594191/train2": 0.5151515151515151, + "2054899126/train0": 0.45, + "2054899126/train1": 0.49122807017543857, + "2054899126/train2": 0.4369747899159664, + "2061635888/train0": 0.6388888888888888, + "2061635888/train1": 0.4722222222222222, + "2061635888/train2": 0.8055555555555556, + "211245842/train0": 0.7058823529411765, + "211245842/train1": 0.8416666666666667, + "211245842/train2": 0.6, + "278008524/train0": 0.5568181818181819, + "278008524/train1": 0.5907608695652173, + "278008524/train2": 0.33611111111111114, + "307068879/train0": 0.42857142857142855, + "307068879/train1": 0.46153846153846156, + "307068879/train2": 0.3626373626373626, + "412102563/train0": 0.35, + "412102563/train1": 0.38461538461538464, + "412102563/train2": 0.6309523809523809, + "440577711/train0": 0.5862068965517241, + "440577711/train1": 0.8666666666666667, + "440577711/train2": 0.7222222222222222, + "441046872/train0": 0.38181818181818183, + "441046872/train1": 0.509090909090909, + "441046872/train2": 0.5636363636363636, + "509430392/train0": 0.5378151260504201, + "509430392/train1": 0.5714285714285714, + "509430392/train2": 0.6302521008403361, + "585904130/train0": 0.44047619047619047, + "585904130/train1": 0.4533333333333333, + "585904130/train2": 0.6818181818181818, + "608016290/train0": 0.5904761904761905, + "608016290/train1": 0.6, + "608016290/train2": 0.44166666666666665, + "626486813/train0": 0.6125, + "626486813/train1": 0.48333333333333334, + "626486813/train2": 0.475, + "636680137/train0": 0.5333333333333333, + "636680137/train1": 0.55, + "636680137/train2": 0.5583333333333333, + "655884605/train0": 0.6388888888888888, + "655884605/train1": 0.4444444444444444, + "655884605/train2": 0.4166666666666667, + "672328262/train0": 0.5294117647058824, + "672328262/train1": 0.6, + "672328262/train2": 0.13333333333333333, + "701667486/train0": 0.5714285714285714, + "701667486/train1": 0.6666666666666666, + "701667486/train2": 0.8888888888888888, + "747260437/train0": 0.38333333333333336, + "747260437/train1": 0.475, + "747260437/train2": 0.45, + "750448296/train0": 0.5869047619047619, + "750448296/train1": 0.4262820512820513, + "750448296/train2": 0.5674603174603174, + "767747424/train0": 0.5088675213675213, + "767747424/train1": 0.43937280040221216, + "767747424/train2": 0.5861649788120377 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5498799072446079, + "low": 0.491791561891809, + "mean": 0.5229314265057493, + "n": 5, + "num_samples": 100 + }, + "mean": 0.5229314265057493, + "per_task": { + "LiftPegUpright-v1": 0.4784086828204475, + "PickCube-v1": 0.5370370370370371, + "PullCube-v1": 0.49303016307826086, + "PushCube-v1": 0.5754796516735593, + "StackCube-v1": 0.5307015979194418 + } + }, + "metric": "pairwise_causal_calibration_accuracy", + "micro": { + "confidence": 0.95, + "high": 0.5361608170491736, + "low": 0.4944078286160967, + "mean": 0.5121602106324892, + "n": 138, + "num_samples": 100 + }, + "n": 138 + }, + "pairwise_causal_calibration_confidence": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5308713827307395, + "low": 0.5272590655822438, + "mean": 0.5293000987125837, + "n": 117, + "num_samples": 100 + }, + "mean": 0.5293000987125837, + "per_seed": { + "1073161778/train0": 0.5251040975256128, + "1073161778/train1": 0.5423238840429205, + "1073161778/train2": 0.5368743958607693, + "1113371261/train0": 0.5379810554394029, + "1113371261/train1": 0.5252684472642476, + "1113371261/train2": 0.5332614138108104, + "1151410346/train0": 0.51710870319415, + "1151410346/train1": 0.5316022421578545, + "1151410346/train2": 0.5177275439295085, + "1261930456/train0": 0.5386347584075308, + "1261930456/train1": 0.5422560685706671, + "1261930456/train2": 0.5426310160661976, + "1324133122/train0": 0.5333367283214293, + "1324133122/train1": 0.5357950970847783, + "1324133122/train2": 0.5266268320102542, + "1324642411/train0": 0.5267509882327646, + "1324642411/train1": 0.5232561219266944, + "1324642411/train2": 0.5209245623930986, + "1375492786/train0": 0.5169520118772317, + "1375492786/train1": 0.5298121533009607, + "1375492786/train2": 0.5313546391394341, + "141984615/train0": 0.5279271325774133, + "141984615/train1": 0.5244535198032577, + "141984615/train2": 0.538380386098592, + "1461069713/train0": 0.5251639529546277, + "1461069713/train1": 0.5166360139443322, + "1461069713/train2": 0.519839782973733, + "1480856810/train0": 0.5226475437853878, + "1480856810/train1": 0.5136404664974943, + "1480856810/train2": 0.5148540856771467, + "1509980664/train0": 0.5158281275240727, + "1509980664/train1": 0.5242300284642385, + "1509980664/train2": 0.5240930381172986, + "1522272686/train0": 0.5457014445746324, + "1522272686/train1": 0.5111398905599821, + "1522272686/train2": 0.525252740289532, + "1549324805/train0": 0.5337482570523958, + "1549324805/train1": 0.5262268456146021, + "1549324805/train2": 0.528533525486533, + "1578990937/train0": 0.5361003534463802, + "1578990937/train1": 0.5394992318577098, + "1578990937/train2": 0.5340698575932721, + "1790188787/train0": 0.5316179775586928, + "1790188787/train1": 0.5328964743528634, + "1790188787/train2": 0.5352532151194661, + "1822210471/train0": 0.539953848754645, + "1822210471/train1": 0.5415632384984795, + "1822210471/train2": 0.5237701807590434, + "1845489013/train0": 0.5171237518534777, + "1845489013/train1": 0.5121094164593272, + "1845489013/train2": 0.5170849899671911, + "1999022574/train0": 0.5267116008364124, + "1999022574/train1": 0.5323449915161098, + "1999022574/train2": 0.5344673895535903, + "2012211287/train0": 0.5305966527751617, + "2012211287/train1": 0.5238503061013462, + "2012211287/train2": 0.5242257655342142, + "2038594191/train0": 0.5768622164838934, + "2038594191/train1": 0.5594515320080404, + "2038594191/train2": 0.5363991279304782, + "2054899126/train0": 0.5138396482606322, + "2054899126/train1": 0.5227936475539869, + "2054899126/train2": 0.5252379082880491, + "2061635888/train0": 0.5184746997192393, + "2061635888/train1": 0.5119131115242276, + "2061635888/train2": 0.5138245455754116, + "211245842/train0": 0.5351161430732934, + "211245842/train1": 0.5382974229033135, + "211245842/train2": 0.5317298952114761, + "278008524/train0": 0.519833289537298, + "278008524/train1": 0.5198421814375511, + "278008524/train2": 0.5255283835156899, + "307068879/train0": 0.5343458881100447, + "307068879/train1": 0.5333254751964351, + "307068879/train2": 0.5472048889164063, + "412102563/train0": 0.5258131375323639, + "412102563/train1": 0.523423235146417, + "412102563/train2": 0.5248002449882088, + "440577711/train0": 0.5272168394784098, + "440577711/train1": 0.5379425484601451, + "440577711/train2": 0.536464978436527, + "441046872/train0": 0.5381289217490534, + "441046872/train1": 0.5333064538046144, + "441046872/train2": 0.5375543539918383, + "509430392/train0": 0.5162619682864167, + "509430392/train1": 0.5280283691474993, + "509430392/train2": 0.5305902079798236, + "585904130/train0": 0.5212774691287471, + "585904130/train1": 0.5302663446880443, + "585904130/train2": 0.522915616847175, + "608016290/train0": 0.5293645053307068, + "608016290/train1": 0.5357192536556177, + "608016290/train2": 0.5315196569218078, + "626486813/train0": 0.5273776063219382, + "626486813/train1": 0.521404963299847, + "626486813/train2": 0.531833628670964, + "636680137/train0": 0.5395580589892535, + "636680137/train1": 0.5207010539912516, + "636680137/train2": 0.5159516842599035, + "655884605/train0": 0.5310489483070069, + "655884605/train1": 0.5070899546645378, + "655884605/train2": 0.5234811460445998, + "672328262/train0": 0.5321127842961938, + "672328262/train1": 0.5540392504779817, + "672328262/train2": 0.5470577940231715, + "701667486/train0": 0.5385441284405091, + "701667486/train1": 0.5308940956422026, + "701667486/train2": 0.5566493339472486, + "747260437/train0": 0.522326045653055, + "747260437/train1": 0.5240130464861401, + "747260437/train2": 0.5277135352836209, + "750448296/train0": 0.5367471317580962, + "750448296/train1": 0.5306744409251447, + "750448296/train2": 0.5335771984593171, + "767747424/train0": 0.5216055060310809, + "767747424/train1": 0.5291623924295586, + "767747424/train2": 0.5228149250617854 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5314758068699093, + "low": 0.5277782420526062, + "mean": 0.5293220940263759, + "n": 5, + "num_samples": 100 + }, + "mean": 0.5293220940263759, + "per_task": { + "LiftPegUpright-v1": 0.5335629666344366, + "PickCube-v1": 0.5273646918080489, + "PullCube-v1": 0.5282915670267949, + "PushCube-v1": 0.528992677243157, + "StackCube-v1": 0.5283985674194418 + } + }, + "metric": "pairwise_causal_calibration_confidence", + "micro": { + "confidence": 0.95, + "high": 0.5304547887963105, + "low": 0.5271199029241195, + "mean": 0.5289614319928112, + "n": 138, + "num_samples": 100 + }, + "n": 138 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1275448588079086, + "low": 0.09033671262605028, + "mean": 0.10743756299718471, + "n": 120, + "num_samples": 100 + }, + "mean": 0.10743756299718471, + "per_seed": { + "1073161778/train0": 0.13854947567687334, + "1073161778/train1": 0.20065721737625392, + "1073161778/train2": 0.3440779850916114, + "1113371261/train0": 0.06571145427295252, + "1113371261/train1": 0.1890172670214667, + "1113371261/train2": 0.0024528719034753177, + "1151410346/train0": 0.10758489367034052, + "1151410346/train1": 0.049350138794526544, + "1151410346/train2": 0.05106087726284181, + "1261930456/train0": 0.03358425335702575, + "1261930456/train1": 0.048653022338423835, + "1261930456/train2": 0.05260707917189755, + "1324133122/train0": 0.14047958546428646, + "1324133122/train1": 0.1786522399419212, + "1324133122/train2": 0.026626832010254176, + "1324642411/train0": 0.043413085733646106, + "1324642411/train1": 0.035861163943501084, + "1324642411/train2": 0.09725725578871958, + "1375492786/train0": 0.023618678543898375, + "1375492786/train1": 0.049135215120092, + "1375492786/train2": 0.09087844866324368, + "141984615/train0": 0.06959379924407999, + "141984615/train1": 0.05054648019674224, + "141984615/train2": 0.043205574824282285, + "1461069713/train0": 0.03349728628796106, + "1461069713/train1": 0.08330268061099888, + "1461069713/train2": 0.08016021702626697, + "1480856810/train0": 0.10850612964397366, + "1480856810/train1": 0.02624550851430102, + "1480856810/train2": 0.07338120844050033, + "1509980664/train0": 0.08249479419073935, + "1509980664/train1": 0.2226427268769369, + "1509980664/train2": 0.012981927006187598, + "1522272686/train0": 0.14874299986981188, + "1522272686/train1": 0.04441566499557348, + "1522272686/train2": 0.16414162917842084, + "1549324805/train0": 0.21625174294760421, + "1549324805/train1": 0.1512268456146021, + "1549324805/train2": 0.2702001921531997, + "1578990937/train0": 0.042038781082623024, + "1578990937/train1": 0.038926080840327754, + "1578990937/train2": 0.1391118743999947, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0956547497140344, + "1790188787/train1": 0.010963174769943684, + "1790188787/train2": 0.0988895787558297, + "1822210471/train0": 0.15111259204903227, + "1822210471/train1": 0.10876536091501116, + "1822210471/train2": 0.026229819240956687, + "1845489013/train0": 0.2562541866360864, + "1845489013/train1": 0.12749403184394253, + "1845489013/train2": 0.002915010032808918, + "1999022574/train0": 0.2695687436935554, + "1999022574/train1": 0.10072366188957278, + "1999022574/train2": 0.031322084130620015, + "2012211287/train0": 0.22807564437180022, + "2012211287/train1": 0.09885030610134621, + "2012211287/train2": 0.024225765534214183, + "2038594191/train0": 0.5768622164838934, + "2038594191/train1": 0.2553362522454674, + "2038594191/train2": 0.03713806757575985, + "2054899126/train0": 0.031919824130316105, + "2054899126/train1": 0.015782788689274185, + "2054899126/train2": 0.044131559186041336, + "2061635888/train0": 0.12041418916964952, + "2061635888/train1": 0.03969088930200537, + "2061635888/train2": 0.291731009980144, + "211245842/train0": 0.17076620986788324, + "211245842/train1": 0.30336924376335306, + "211245842/train2": 0.06827010478852391, + "278008524/train0": 0.13700175449974886, + "278008524/train1": 0.1186012755305571, + "278008524/train2": 0.1894172724045787, + "307068879/train0": 0.10577445953861614, + "307068879/train1": 0.07178701365797358, + "307068879/train2": 0.1845675262790436, + "412102563/train0": 0.17581313753236394, + "412102563/train1": 0.13880785053103237, + "412102563/train2": 0.12044288641048877, + "440577711/train0": 0.058990057073314284, + "440577711/train1": 0.3287241182065216, + "440577711/train2": 0.18575724378569516, + "441046872/train0": 0.15631073993087152, + "441046872/train1": 0.02421554471370535, + "441046872/train2": 0.03382332741347877, + "509430392/train0": 0.02155315776400346, + "509430392/train1": 0.04340020228107211, + "509430392/train2": 0.09966189286051252, + "585904130/train0": 0.0808012786525566, + "585904130/train1": 0.076933011354711, + "585904130/train2": 0.15890256497100674, + "608016290/train0": 0.14243134895449866, + "608016290/train1": 0.0682003715539854, + "608016290/train2": 0.08985299025514107, + "626486813/train0": 0.08512239367806185, + "626486813/train1": 0.03807162996651359, + "626486813/train2": 0.056833628670964004, + "636680137/train0": 0.006224725655920161, + "636680137/train1": 0.029298946008748494, + "636680137/train2": 0.042381649073429895, + "655884605/train0": 0.10783994058188195, + "655884605/train1": 0.06264551022009335, + "655884605/train2": 0.10681447937793315, + "672328262/train0": 0.0027010195903114376, + "672328262/train1": 0.06500635112191766, + "672328262/train2": 0.41372446068983815, + "701667486/train0": 0.06175922923362397, + "701667486/train1": 0.1357725710244641, + "701667486/train2": 0.3322395549416404, + "747260437/train0": 0.1389927123197216, + "747260437/train1": 0.04901304648614013, + "747260437/train2": 0.07771353528362085, + "750448296/train0": 0.07635409425053818, + "750448296/train1": 0.13910389780114304, + "750448296/train2": 0.14091077521706144, + "767747424/train0": 0.08288086769862717, + "767747424/train1": 0.08978959202734649, + "767747424/train2": 0.0822416726337075 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.12179850626790115, + "low": 0.09390325784867848, + "mean": 0.10937426577475251, + "n": 5, + "num_samples": 100 + }, + "mean": 0.10937426577475251, + "per_task": { + "LiftPegUpright-v1": 0.1274266477632942, + "PickCube-v1": 0.11910009801460207, + "PullCube-v1": 0.09823916112294553, + "PushCube-v1": 0.08128349345238421, + "StackCube-v1": 0.12082192852053648 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.1178502721817909, + "low": 0.09189770860488271, + "mean": 0.10476988623635247, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "pairwise_causal_calibration_pairs": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 88.09791666666666, + "low": 75.28541666666666, + "mean": 82.21180555555556, + "n": 120, + "num_samples": 100 + }, + "mean": 82.21180555555556, + "per_seed": { + "1073161778/train0": 119.0, + "1073161778/train1": 120.0, + "1073161778/train2": 84.0, + "1113371261/train0": 28.0, + "1113371261/train1": 28.0, + "1113371261/train2": 28.0, + "1151410346/train0": 105.0, + "1151410346/train1": 105.0, + "1151410346/train2": 105.0, + "1261930456/train0": 99.0, + "1261930456/train1": 110.0, + "1261930456/train2": 84.0, + "1324133122/train0": 28.0, + "1324133122/train1": 28.0, + "1324133122/train2": 28.0, + "1324642411/train0": 120.0, + "1324642411/train1": 119.0, + "1324642411/train2": 110.0, + "1375492786/train0": 75.0, + "1375492786/train1": 114.0, + "1375492786/train2": 84.0, + "141984615/train0": 120.0, + "141984615/train1": 120.0, + "141984615/train2": 120.0, + "1461069713/train0": 120.0, + "1461069713/train1": 120.0, + "1461069713/train2": 120.0, + "1480856810/train0": 99.0, + "1480856810/train1": 119.0, + "1480856810/train2": 119.0, + "1509980664/train0": 90.0, + "1509980664/train1": 63.0, + "1509980664/train2": 90.0, + "1522272686/train0": 36.0, + "1522272686/train1": 36.0, + "1522272686/train2": 36.0, + "1549324805/train0": 120.0, + "1549324805/train1": 120.0, + "1549324805/train2": 120.0, + "1578990937/train0": 117.0, + "1578990937/train1": 110.0, + "1578990937/train2": 119.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 110.0, + "1790188787/train1": 114.0, + "1790188787/train2": 110.0, + "1822210471/train0": 40.0, + "1822210471/train1": 49.0, + "1822210471/train2": 40.0, + "1845489013/train0": 46.0, + "1845489013/train1": 13.0, + "1845489013/train2": 25.0, + "1999022574/train0": 70.0, + "1999022574/train1": 70.0, + "1999022574/train2": 76.0, + "2012211287/train0": 119.0, + "2012211287/train1": 120.0, + "2012211287/train2": 92.0, + "2038594191/train0": 29.0, + "2038594191/train1": 54.0, + "2038594191/train2": 99.0, + "2054899126/train0": 60.0, + "2054899126/train1": 57.0, + "2054899126/train2": 59.5, + "2061635888/train0": 36.0, + "2061635888/train1": 36.0, + "2061635888/train2": 36.0, + "211245842/train0": 119.0, + "211245842/train1": 120.0, + "211245842/train2": 120.0, + "278008524/train0": 109.5, + "278008524/train1": 106.0, + "278008524/train2": 109.5, + "307068879/train0": 91.0, + "307068879/train1": 91.0, + "307068879/train2": 91.0, + "412102563/train0": 60.0, + "412102563/train1": 39.0, + "412102563/train2": 84.0, + "440577711/train0": 29.0, + "440577711/train1": 15.0, + "440577711/train2": 54.0, + "441046872/train0": 55.0, + "441046872/train1": 55.0, + "441046872/train2": 55.0, + "509430392/train0": 119.0, + "509430392/train1": 105.0, + "509430392/train2": 119.0, + "585904130/train0": 84.0, + "585904130/train1": 75.0, + "585904130/train2": 110.0, + "608016290/train0": 105.0, + "608016290/train1": 105.0, + "608016290/train2": 120.0, + "626486813/train0": 120.0, + "626486813/train1": 120.0, + "626486813/train2": 120.0, + "636680137/train0": 120.0, + "636680137/train1": 120.0, + "636680137/train2": 120.0, + "655884605/train0": 36.0, + "655884605/train1": 36.0, + "655884605/train2": 36.0, + "672328262/train0": 17.0, + "672328262/train1": 35.0, + "672328262/train2": 30.0, + "701667486/train0": 42.0, + "701667486/train1": 84.0, + "701667486/train2": 54.0, + "747260437/train0": 120.0, + "747260437/train1": 120.0, + "747260437/train2": 120.0, + "750448296/train0": 108.0, + "750448296/train1": 101.66666666666667, + "750448296/train2": 108.0, + "767747424/train0": 119.25, + "767747424/train1": 119.0, + "767747424/train2": 119.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 94.86577777777778, + "low": 53.554909090909085, + "mean": 77.60379797979797, + "n": 5, + "num_samples": 100 + }, + "mean": 77.60379797979797, + "per_task": { + "LiftPegUpright-v1": 94.8, + "PickCube-v1": 36.0, + "PullCube-v1": 92.32, + "PushCube-v1": 97.44444444444444, + "StackCube-v1": 67.45454545454545 + } + }, + "metric": "pairwise_causal_calibration_pairs", + "micro": { + "confidence": 0.95, + "high": 92.92361111111111, + "low": 81.02083333333333, + "mean": 86.34722222222223, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1549963900908925, + "low": 0.05221239623515059, + "mean": 0.09457998078331692, + "n": 120, + "num_samples": 100 + }, + "mean": 0.09457998078331692, + "per_seed": { + "1073161778/train0": 0.2551750656075202, + "1073161778/train1": 0.20299343323495855, + "1073161778/train2": 0.24613098923868526, + "1113371261/train0": 0.15799813692233977, + "1113371261/train1": 0.13463328766802296, + "1113371261/train2": -0.0627076273903946, + "1151410346/train0": -0.030788448510796762, + "1151410346/train1": -0.05730648283766754, + "1151410346/train2": -0.033176433226523416, + "1261930456/train0": 0.3447490352381858, + "1261930456/train1": 0.2330187222265443, + "1261930456/train2": 0.3117657280355065, + "1324133122/train0": -0.1742924240977557, + "1324133122/train1": 0.18792055489105874, + "1324133122/train2": -0.1739828724101382, + "1324642411/train0": 0.6155609171443972, + "1324642411/train1": 0.24134426910254947, + "1324642411/train2": -0.06739236477010135, + "1375492786/train0": -0.07560538396524885, + "1375492786/train1": -0.4256345336980503, + "1375492786/train2": -0.3658750831677004, + "141984615/train0": 0.008737131332534287, + "141984615/train1": 0.11792927817531362, + "141984615/train2": 0.034121695817442343, + "1461069713/train0": 0.4675581452853047, + "1461069713/train1": 0.4500223738950829, + "1461069713/train2": 0.4462633698719307, + "1480856810/train0": 0.6476723025740536, + "1480856810/train1": 0.38625575833343895, + "1480856810/train2": 0.35255742400707646, + "1509980664/train0": -0.12116588314285559, + "1509980664/train1": -0.10941040668136354, + "1509980664/train2": -0.14659151504624657, + "1522272686/train0": -0.23967836808523524, + "1522272686/train1": -0.2514915262180857, + "1522272686/train2": -0.28676975669762994, + "1549324805/train0": 0.5713459023104054, + "1549324805/train1": 0.3107421704514528, + "1549324805/train2": 0.6769572594134817, + "1578990937/train0": -0.11714129730488587, + "1578990937/train1": -0.09068349284851132, + "1578990937/train2": -0.14154551725382616, + "1747178030/train0": 0.3567590128192366, + "1747178030/train1": 0.19263300195342614, + "1747178030/train2": 0.2551836974507351, + "1790188787/train0": 0.5329093436927219, + "1790188787/train1": 0.5122492091762928, + "1790188787/train2": 0.4672762110936981, + "1822210471/train0": -0.05440518485836532, + "1822210471/train1": -0.03958497742915945, + "1822210471/train2": -0.16858629361831445, + "1845489013/train0": -0.13774783214464598, + "1845489013/train1": -0.07780183911371158, + "1845489013/train2": -0.12444815357019567, + "1999022574/train0": -0.11408015116078918, + "1999022574/train1": -0.17466576670619666, + "1999022574/train2": -0.16215465004219584, + "2012211287/train0": 0.2180007600904328, + "2012211287/train1": 0.16726605949103523, + "2012211287/train2": 0.24853085052742357, + "2038594191/train0": 0.08574322718445004, + "2038594191/train1": 0.1354107312118617, + "2038594191/train2": 0.08745956040002545, + "2054899126/train0": 0.2227645387030986, + "2054899126/train1": 0.2770881776119133, + "2054899126/train2": 0.1326005713312028, + "2061635888/train0": 0.13802527783024798, + "2061635888/train1": 0.0925196999812683, + "2061635888/train2": 0.061796487673025036, + "211245842/train0": -0.16494316577956913, + "211245842/train1": -0.10202293411527177, + "211245842/train2": -0.26370842901349006, + "278008524/train0": 0.1024719525224913, + "278008524/train1": 0.16388738649092643, + "278008524/train2": 0.16857295234438457, + "307068879/train0": -0.3084681084329295, + "307068879/train1": -0.22321666120891376, + "307068879/train2": -0.04845957189152705, + "412102563/train0": -0.10215255174920451, + "412102563/train1": -0.15959785934997645, + "412102563/train2": -0.20549642192166445, + "440577711/train0": 0.7070441087351717, + "440577711/train1": 0.7017571830152312, + "440577711/train2": 0.6922087959324436, + "441046872/train0": 0.009282072163031563, + "441046872/train1": -0.11963786997800063, + "441046872/train2": -0.08224411989102232, + "509430392/train0": 0.5136988024654346, + "509430392/train1": 0.4261006779620051, + "509430392/train2": 0.3587989801510558, + "585904130/train0": 0.08651310891179698, + "585904130/train1": 0.09881997958173372, + "585904130/train2": 0.0589756379475741, + "608016290/train0": 0.08072228079745068, + "608016290/train1": -0.1274552507433626, + "608016290/train2": -0.030776181488394214, + "626486813/train0": 0.34770303375956, + "626486813/train1": 0.2450842010496616, + "626486813/train2": 0.22550405863537087, + "636680137/train0": 0.8489010665175709, + "636680137/train1": 0.39916002861669686, + "636680137/train2": 0.47473165550706736, + "655884605/train0": 0.09399166521150604, + "655884605/train1": 0.07407655216726083, + "655884605/train2": 0.07294104079981856, + "672328262/train0": -0.11406027153604881, + "672328262/train1": -0.16708550898936614, + "672328262/train2": -0.1772708238485603, + "701667486/train0": -0.0742733989521003, + "701667486/train1": -0.061055849565622045, + "701667486/train2": -0.00871862718880162, + "747260437/train0": -0.04417027513292293, + "747260437/train1": -0.15678779044109292, + "747260437/train2": 0.08370080144449699, + "750448296/train0": -0.04826188693738287, + "750448296/train1": -0.20270377858890484, + "750448296/train2": -0.10917174959959015, + "767747424/train0": -0.06789015981965127, + "767747424/train1": -0.004134822890086619, + "767747424/train2": -0.07224136268104218 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.27892833673762724, + "low": -0.16022128328270682, + "mean": 0.048763676236597255, + "n": 5, + "num_samples": 100 + }, + "mean": 0.048763676236597255, + "per_task": { + "LiftPegUpright-v1": 0.03252291650109252, + "PickCube-v1": -0.25931321700031695, + "PullCube-v1": 0.0830997467005694, + "PushCube-v1": 0.44319861689531703, + "StackCube-v1": -0.05568968191367572 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.13903211305326382, + "low": 0.03678310252525295, + "mean": 0.08390417152793436, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6652777777777779, + "low": 0.5138888888888888, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 100 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7563636363636363, + "low": 0.20969696969696966, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 100 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6319444444444444, + "low": 0.5, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4011238396594614, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 100 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5757816767742236, + "low": 0.5432721244566368, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 100 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.3363403293511107, + "low": 1.0818447619159188, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 7.079166666666667, + "low": 5.741666666666666, + "mean": 6.384027777777778, + "n": 120, + "num_samples": 100 + }, + "mean": 6.384027777777778, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 8.0, + "1073161778/train2": 6.0, + "1113371261/train0": 1.0, + "1113371261/train1": 2.0, + "1113371261/train2": 0.0, + "1151410346/train0": 12.0, + "1151410346/train1": 7.0, + "1151410346/train2": 6.0, + "1261930456/train0": 6.0, + "1261930456/train1": 10.0, + "1261930456/train2": 6.0, + "1324133122/train0": 7.0, + "1324133122/train1": 1.0, + "1324133122/train2": 7.0, + "1324642411/train0": 7.0, + "1324642411/train1": 11.0, + "1324642411/train2": 5.0, + "1375492786/train0": 0.0, + "1375492786/train1": 11.0, + "1375492786/train2": 9.0, + "141984615/train0": 7.0, + "141984615/train1": 15.0, + "141984615/train2": 11.0, + "1461069713/train0": 7.0, + "1461069713/train1": 3.0, + "1461069713/train2": 5.0, + "1480856810/train0": 9.0, + "1480856810/train1": 8.0, + "1480856810/train2": 1.0, + "1509980664/train0": 10.0, + "1509980664/train1": 12.0, + "1509980664/train2": 9.0, + "1522272686/train0": 7.0, + "1522272686/train1": 0.0, + "1522272686/train2": 4.0, + "1549324805/train0": 5.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 4.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 15.0, + "1747178030/train1": 10.0, + "1747178030/train2": 15.0, + "1790188787/train0": 5.0, + "1790188787/train1": 6.0, + "1790188787/train2": 8.0, + "1822210471/train0": 4.0, + "1822210471/train1": 4.0, + "1822210471/train2": 5.0, + "1845489013/train0": 11.0, + "1845489013/train1": 1.0, + "1845489013/train2": 7.0, + "1999022574/train0": 5.0, + "1999022574/train1": 8.0, + "1999022574/train2": 13.0, + "2012211287/train0": 13.0, + "2012211287/train1": 13.0, + "2012211287/train2": 1.0, + "2038594191/train0": 5.0, + "2038594191/train1": 9.0, + "2038594191/train2": 3.0, + "2054899126/train0": 1.5, + "2054899126/train1": 4.0, + "2054899126/train2": 1.0, + "2061635888/train0": 6.0, + "2061635888/train1": 3.0, + "2061635888/train2": 5.0, + "211245842/train0": 10.0, + "211245842/train1": 13.0, + "211245842/train2": 0.0, + "278008524/train0": 8.0, + "278008524/train1": 5.5, + "278008524/train2": 3.0, + "307068879/train0": 11.0, + "307068879/train1": 8.0, + "307068879/train2": 6.0, + "412102563/train0": 7.0, + "412102563/train1": 14.0, + "412102563/train2": 10.0, + "440577711/train0": 0.0, + "440577711/train1": 14.0, + "440577711/train2": 6.0, + "441046872/train0": 2.0, + "441046872/train1": 4.0, + "441046872/train2": 5.0, + "509430392/train0": 11.0, + "509430392/train1": 2.0, + "509430392/train2": 10.0, + "585904130/train0": 4.0, + "585904130/train1": 15.0, + "585904130/train2": 0.0, + "608016290/train0": 11.0, + "608016290/train1": 8.0, + "608016290/train2": 8.0, + "626486813/train0": 6.5, + "626486813/train1": 7.0, + "626486813/train2": 9.5, + "636680137/train0": 2.0, + "636680137/train1": 1.0, + "636680137/train2": 3.0, + "655884605/train0": 0.0, + "655884605/train1": 8.0, + "655884605/train2": 8.0, + "672328262/train0": 5.0, + "672328262/train1": 7.0, + "672328262/train2": 4.0, + "701667486/train0": 2.0, + "701667486/train1": 10.0, + "701667486/train2": 8.0, + "747260437/train0": 3.0, + "747260437/train1": 10.0, + "747260437/train2": 2.0, + "750448296/train0": 5.0, + "750448296/train1": 11.333333333333334, + "750448296/train2": 5.0, + "767747424/train0": 5.5, + "767747424/train1": 6.5, + "767747424/train2": 9.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 6.918181818181819, + "low": 4.7696969696969695, + "mean": 6.105737373737374, + "n": 5, + "num_samples": 100 + }, + "mean": 6.105737373737374, + "per_task": { + "LiftPegUpright-v1": 7.333333333333333, + "PickCube-v1": 3.6666666666666665, + "PullCube-v1": 6.16, + "PushCube-v1": 6.944444444444445, + "StackCube-v1": 6.424242424242424 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 7.0625, + "low": 5.486111111111111, + "mean": 6.388888888888889, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3645833333333333, + "low": 0.21875, + "mean": 0.2916666666666667, + "n": 120, + "num_samples": 100 + }, + "mean": 0.2916666666666667, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.352, + "low": 0.08484848484848485, + "mean": 0.21664646464646461, + "n": 5, + "num_samples": 100 + }, + "mean": 0.21664646464646461, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29333333333333333, + "PushCube-v1": 0.4444444444444444, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3333333333333333, + "low": 0.20833333333333334, + "mean": 0.2708333333333333, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.052083333333333336, + "low": -0.08958333333333333, + "mean": -0.014583333333333334, + "n": 120, + "num_samples": 100 + }, + "mean": -0.014583333333333334, + "per_seed": { + "1073161778/train0": -1.0, + "1073161778/train1": -1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": -1.0, + "1999022574/train1": 0.0, + "1999022574/train2": -1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.0, + "2038594191/train1": -1.0, + "2038594191/train2": -1.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": -1.0, + "412102563/train1": -1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": -1.0, + "747260437/train1": 0.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": -0.25, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1412121212121212, + "low": -0.06545454545454546, + "mean": 0.01676767676767677, + "n": 5, + "num_samples": 100 + }, + "mean": 0.01676767676767677, + "per_task": { + "LiftPegUpright-v1": -0.06666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": -0.06666666666666667, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": -0.06060606060606061 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.06944444444444445, + "low": -0.09722222222222222, + "mean": -0.020833333333333332, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8801274286031711, + "low": 0.6161422166447866, + "mean": 0.7367921772068561, + "n": 120, + "num_samples": 100 + }, + "mean": 0.7367921772068561, + "per_seed": { + "1073161778/train0": 0.67014479637146, + "1073161778/train1": 0.6947963237762451, + "1073161778/train2": 2.0, + "1113371261/train0": 0.09047546982765198, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.14433109760284424, + "1151410346/train0": 0.15656626224517822, + "1151410346/train1": 0.8148510456085205, + "1151410346/train2": 0.18784314393997192, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 0.08501242101192474, + "1324133122/train0": 0.028782829642295837, + "1324133122/train1": 0.06454251706600189, + "1324133122/train2": 0.13575458526611328, + "1324642411/train0": 0.054066285490989685, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.007268309593200684, + "141984615/train1": 0.030119141563773155, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.02898651361465454, + "1461069713/train1": 0.009051382541656494, + "1461069713/train2": 0.06987287849187851, + "1480856810/train0": 0.07608599960803986, + "1480856810/train1": 0.19651852548122406, + "1480856810/train2": 0.21615806221961975, + "1509980664/train0": 2.0, + "1509980664/train1": 0.8129013180732727, + "1509980664/train2": 0.16154873371124268, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.03462250903248787, + "1549324805/train1": 0.0303483996540308, + "1549324805/train2": 0.04690708592534065, + "1578990937/train0": 0.14173981547355652, + "1578990937/train1": 2.0, + "1578990937/train2": 0.1460946798324585, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.13741976022720337, + "1790188787/train2": 0.13743820786476135, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 0.14746195077896118, + "1999022574/train1": 2.0, + "1999022574/train2": 0.8302689790725708, + "2012211287/train0": 0.7026570439338684, + "2012211287/train1": 0.6900332570075989, + "2012211287/train2": 0.6829424500465393, + "2038594191/train0": 0.04285776615142822, + "2038594191/train1": 0.07674666494131088, + "2038594191/train2": 0.12977690994739532, + "2054899126/train0": 1.0391597636044025, + "2054899126/train1": 1.0660420358181, + "2054899126/train2": 1.0276100430637598, + "2061635888/train0": 0.043751150369644165, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 0.12586313486099243, + "211245842/train1": 0.17156320810317993, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.06754684634506702, + "278008524/train1": 1.0095977680757642, + "278008524/train2": 1.034129787236452, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.20841218531131744, + "412102563/train0": 0.14946338534355164, + "412102563/train1": 0.8125194311141968, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.20389758050441742, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 0.09919407218694687, + "585904130/train1": 0.04792279005050659, + "585904130/train2": 0.08930286020040512, + "608016290/train0": 2.0, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.21245408058166504, + "626486813/train0": 0.011467556469142437, + "626486813/train1": 0.015682418830692768, + "626486813/train2": 0.030346781481057405, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.017789265140891075, + "636680137/train2": 0.03889387845993042, + "655884605/train0": 0.12222731113433838, + "655884605/train1": 0.12105989456176758, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.04841095209121704, + "747260437/train1": 2.0, + "747260437/train2": 0.06004788354039192, + "750448296/train0": 0.08246687302986781, + "750448296/train1": 0.10720233743389447, + "750448296/train2": 0.11559025570750237, + "767747424/train0": 0.5665738391689956, + "767747424/train1": 0.07074803306022659, + "767747424/train2": 0.6022169017232955 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8493481941850234, + "low": 0.5996402573417872, + "mean": 0.7081122018725922, + "n": 5, + "num_samples": 100 + }, + "mean": 0.7081122018725922, + "per_task": { + "LiftPegUpright-v1": 0.7594087203343709, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.6347246278480937, + "PushCube-v1": 0.9508692075808843, + "StackCube-v1": 0.6115993509689966 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.7897342939654158, + "low": 0.5671833170215703, + "mean": 0.6808734688885225, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.06579907784681484, + "low": -0.19408030954073185, + "mean": -0.05332588509093815, + "n": 120, + "num_samples": 100 + }, + "mean": -0.05332588509093815, + "per_seed": { + "1073161778/train0": -1.32985520362854, + "1073161778/train1": -1.3052036762237549, + "1073161778/train2": 0.0, + "1113371261/train0": -0.06794795393943787, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": -0.014092326164245605, + "1151410346/train0": -0.06943503022193909, + "1151410346/train1": 0.5888497531414032, + "1151410346/train2": -0.038158148527145386, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.9149875789880753, + "1324133122/train0": -0.06342208385467529, + "1324133122/train1": -0.02766239643096924, + "1324133122/train2": 0.04354967176914215, + "1324642411/train0": -0.052404239773750305, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.09556593745946884, + "141984615/train1": -0.07271510548889637, + "141984615/train2": -0.04388253018260002, + "1461069713/train0": -0.06491655111312866, + "1461069713/train1": -0.08485168218612671, + "1461069713/train2": -0.024030186235904694, + "1480856810/train0": -0.010948076844215393, + "1480856810/train1": 0.10948444902896881, + "1480856810/train2": 0.1291239857673645, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": -0.02050870656967163, + "1509980664/train2": -0.6718612909317017, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.06989232823252678, + "1549324805/train1": -0.07416643761098385, + "1549324805/train2": -0.057607751339673996, + "1578990937/train0": -0.6823966801166534, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": -0.6780418157577515, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.87523253262043, + "1790188787/train1": 0.012652292847633362, + "1790188787/train2": 0.012670740485191345, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": -1.8525380492210388, + "1999022574/train1": 0.0, + "1999022574/train2": -1.1697310209274292, + "2012211287/train0": -0.020948290824890137, + "2012211287/train1": -0.03357207775115967, + "2012211287/train2": -0.04066288471221924, + "2038594191/train0": -1.9571422338485718, + "2038594191/train1": -1.9232533350586891, + "2038594191/train2": -1.8702230900526047, + "2054899126/train0": -0.9608402363955975, + "2054899126/train1": -0.9339579641819, + "2054899126/train2": -0.9723899569362402, + "2061635888/train0": -0.008499354124069214, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": -0.06504528224468231, + "211245842/train1": -0.019345209002494812, + "211245842/train2": 0.0, + "278008524/train0": 0.02481185644865036, + "278008524/train1": 0.9668627781793475, + "278008524/train2": 0.9913947973400354, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": -0.3916783779859543, + "412102563/train0": -1.8505366146564484, + "412102563/train1": -1.1874805688858032, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": -0.49605758488178253, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.999999776482582, + "509430392/train1": 1.999999776482582, + "509430392/train2": 5.200505256652832e-06, + "585904130/train0": -0.0825214758515358, + "585904130/train1": -0.13379275798797607, + "585904130/train2": -0.09241268783807755, + "608016290/train0": 1.274298369884491, + "608016290/train1": 0.0, + "608016290/train2": -0.513247549533844, + "626486813/train0": 0.006369213748257607, + "626486813/train1": 0.010584076109807938, + "626486813/train2": 0.025248438760172576, + "636680137/train0": 0.005126260221004486, + "636680137/train1": -0.0653457809239626, + "636680137/train2": -0.04424116760492325, + "655884605/train0": -0.040779367089271545, + "655884605/train1": -0.041946783661842346, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": -1.951589047908783, + "747260437/train1": 0.0, + "747260437/train2": -1.939952116459608, + "750448296/train0": -0.035126276314258575, + "750448296/train1": -0.010390811910231909, + "750448296/train2": -0.002002893636624018, + "767747424/train0": -0.006625502835959196, + "767747424/train1": -0.5024513089447282, + "767747424/train2": 0.029017559718340635 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2688974869310235, + "low": -0.1753253284128044, + "mean": 0.012552404698577102, + "n": 5, + "num_samples": 100 + }, + "mean": 0.012552404698577102, + "per_task": { + "LiftPegUpright-v1": -0.19046181837717693, + "PickCube-v1": 0.0, + "PullCube-v1": -0.14044583337536704, + "PushCube-v1": 0.5584650288025538, + "StackCube-v1": -0.16479535355712427 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.09707772356988345, + "low": -0.1951776738302821, + "mean": -0.060946117553814676, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6946892382072595, + "low": 0.4387759669192342, + "mean": 0.5455100193744227, + "n": 120, + "num_samples": 100 + }, + "mean": 0.5455100193744227, + "per_seed": { + "1073161778/train0": 1.32985520362854, + "1073161778/train1": 1.3052036762237549, + "1073161778/train2": 0.0, + "1113371261/train0": 0.012273147702217102, + "1113371261/train1": 0.0, + "1113371261/train2": 0.05094057321548462, + "1151410346/train0": 0.7097858786582947, + "1151410346/train1": 1.1851489543914795, + "1151410346/train2": 0.36436158418655396, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.9149875789880753, + "1324133122/train0": 0.026206374168395996, + "1324133122/train1": 0.067916139960289, + "1324133122/train2": 0.05715706944465637, + "1324642411/train0": 0.08999375998973846, + "1324642411/train1": 1.89352947473526, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.007909595966339111, + "141984615/train1": 0.006891986355185509, + "141984615/train2": 0.0, + "1461069713/train0": 0.03121185302734375, + "1461069713/train1": 0.007533630356192589, + "1461069713/train2": 0.03010360151529312, + "1480856810/train0": 1.9239140003919601, + "1480856810/train1": 1.803481474518776, + "1480856810/train2": 1.7838419377803802, + "1509980664/train0": 0.0, + "1509980664/train1": 1.1870986819267273, + "1509980664/train2": 1.8384512662887573, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.026248913258314133, + "1549324805/train1": 0.16857311688363552, + "1549324805/train2": 0.06619475409388542, + "1578990937/train0": 1.8582601845264435, + "1578990937/train1": 0.0, + "1578990937/train2": 1.8539053201675415, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8625802397727966, + "1790188787/train2": 1.8625617921352386, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.8525380492210388, + "1999022574/train1": 0.0, + "1999022574/train2": 1.1697310209274292, + "2012211287/train0": 1.2973429560661316, + "2012211287/train1": 1.3099667429924011, + "2012211287/train2": 1.3170575499534607, + "2038594191/train0": 1.9571422338485718, + "2038594191/train1": 1.9232533350586891, + "2038594191/train2": 1.8702230900526047, + "2054899126/train0": 0.03812338784337044, + "2054899126/train1": 0.9339579641819, + "2054899126/train2": 0.9723899569362402, + "2061635888/train0": 0.054248154163360596, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.8741368651390076, + "211245842/train1": 0.04139396548271179, + "211245842/train2": 0.04297435283660889, + "278008524/train0": 1.0058094765990973, + "278008524/train1": 0.028076826594769955, + "278008524/train2": 0.10181912407279015, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.021801263093948364, + "412102563/train0": 1.8505366146564484, + "412102563/train1": 1.1874805688858032, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.7961024194955826, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.9008059278130531, + "585904130/train1": 1.9520772099494934, + "585904130/train2": 1.9106971397995949, + "608016290/train0": 0.0, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.49927353858947754, + "626486813/train0": 0.0332279191352427, + "626486813/train1": 0.020145455840975046, + "626486813/train2": 0.014552473556250334, + "636680137/train0": 0.0, + "636680137/train1": 0.04962191917002201, + "636680137/train2": 0.03675774484872818, + "655884605/train0": 0.015101373195648193, + "655884605/train1": 0.6918159127235413, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.059984683990478516, + "747260437/train1": 0.0, + "747260437/train2": 0.06505399569869041, + "750448296/train0": 0.670193454871575, + "750448296/train1": 1.2633009366691113, + "750448296/train2": 1.23485841229558, + "767747424/train0": 0.4758553486317396, + "767747424/train1": 0.9922050581662916, + "767747424/train2": 0.46495019318535924 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7369449271975707, + "low": 0.21807832790143564, + "mean": 0.5120192081916777, + "n": 5, + "num_samples": 100 + }, + "mean": 0.5120192081916777, + "per_task": { + "LiftPegUpright-v1": 0.6795656750599544, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5004223387533178, + "PushCube-v1": 0.8349122073915269, + "StackCube-v1": 0.5451958197535891 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.6658392620027169, + "low": 0.42901301130122094, + "mean": 0.5607294605369033, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.23472222222222222, + "mean": 0.3013888888888889, + "n": 120, + "num_samples": 100 + }, + "mean": 0.3013888888888889, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.41955555555555557, + "low": 0.12121212121212122, + "mean": 0.2934949494949495, + "n": 5, + "num_samples": 100 + }, + "mean": 0.2934949494949495, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.25333333333333335, + "PushCube-v1": 0.4444444444444444, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2361111111111111, + "mean": 0.3055555555555556, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3097222222222223, + "low": 0.15416666666666667, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 100 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.776969696969697, + "low": 0.152, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 100 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2569444444444444, + "low": 0.1527777777777778, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5472370309354221, + "low": 0.2993064477612885, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 100 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.4056389139457182, + "low": 0.2675956385682027, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 100 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.4778769724071026, + "low": 0.284725752296961, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 100 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.013112044560281758, + "low": -0.07392049665684591, + "mean": -0.04738573531976173, + "n": 120, + "num_samples": 100 + }, + "mean": -0.04738573531976173, + "per_seed": { + "1073161778/train0": -0.07385947182024077, + "1073161778/train1": -0.07385947182024077, + "1073161778/train2": -0.07385947182024077, + "1113371261/train0": -0.18710182505305797, + "1113371261/train1": -0.18710182505305797, + "1113371261/train2": -0.18710182505305797, + "1151410346/train0": -0.18710182505305797, + "1151410346/train1": -0.18710182505305797, + "1151410346/train2": -0.18710182505305797, + "1261930456/train0": 0.28744401244938833, + "1261930456/train1": 0.28744401244938833, + "1261930456/train2": 0.28744401244938833, + "1324133122/train0": -0.18710182505305797, + "1324133122/train1": -0.18710182505305797, + "1324133122/train2": -0.18710182505305797, + "1324642411/train0": 0.28744401244938833, + "1324642411/train1": 0.28744401244938833, + "1324642411/train2": 0.28744401244938833, + "1375492786/train0": -0.0718611703384171, + "1375492786/train1": -0.0718611703384171, + "1375492786/train2": -0.0718611703384171, + "141984615/train0": -0.0718611703384171, + "141984615/train1": -0.0718611703384171, + "141984615/train2": -0.0718611703384171, + "1461069713/train0": -0.0718611703384171, + "1461069713/train1": -0.0718611703384171, + "1461069713/train2": -0.0718611703384171, + "1480856810/train0": 0.28744401244938833, + "1480856810/train1": 0.28744401244938833, + "1480856810/train2": 0.28744401244938833, + "1509980664/train0": -0.18710182505305797, + "1509980664/train1": -0.18710182505305797, + "1509980664/train2": -0.18710182505305797, + "1522272686/train0": 0.02896384295113541, + "1522272686/train1": 0.02896384295113541, + "1522272686/train2": 0.02896384295113541, + "1549324805/train0": -0.0718611703384171, + "1549324805/train1": -0.0718611703384171, + "1549324805/train2": -0.0718611703384171, + "1578990937/train0": -0.18710182505305797, + "1578990937/train1": -0.18710182505305797, + "1578990937/train2": -0.18710182505305797, + "1747178030/train0": -0.0718611703384171, + "1747178030/train1": -0.0718611703384171, + "1747178030/train2": -0.0718611703384171, + "1790188787/train0": 0.28744401244938833, + "1790188787/train1": 0.28744401244938833, + "1790188787/train2": 0.28744401244938833, + "1822210471/train0": -0.0718611703384171, + "1822210471/train1": -0.0718611703384171, + "1822210471/train2": -0.0718611703384171, + "1845489013/train0": -0.0718611703384171, + "1845489013/train1": -0.0718611703384171, + "1845489013/train2": -0.0718611703384171, + "1999022574/train0": -0.18710182505305797, + "1999022574/train1": -0.18710182505305797, + "1999022574/train2": -0.18710182505305797, + "2012211287/train0": -0.07385947182024077, + "2012211287/train1": -0.07385947182024077, + "2012211287/train2": -0.07385947182024077, + "2038594191/train0": -0.0718611703384171, + "2038594191/train1": -0.0718611703384171, + "2038594191/train2": -0.0718611703384171, + "2054899126/train0": -0.0718611703384171, + "2054899126/train1": -0.0718611703384171, + "2054899126/train2": -0.0718611703384171, + "2061635888/train0": -0.18710182505305797, + "2061635888/train1": -0.18710182505305797, + "2061635888/train2": -0.18710182505305797, + "211245842/train0": -0.18710182505305797, + "211245842/train1": -0.18710182505305797, + "211245842/train2": -0.18710182505305797, + "278008524/train0": -0.0718611703384171, + "278008524/train1": -0.0718611703384171, + "278008524/train2": -0.0718611703384171, + "307068879/train0": -0.07385947182024077, + "307068879/train1": -0.07385947182024077, + "307068879/train2": -0.07385947182024077, + "412102563/train0": -0.18710182505305797, + "412102563/train1": -0.18710182505305797, + "412102563/train2": -0.18710182505305797, + "440577711/train0": 0.28744401244938833, + "440577711/train1": 0.28744401244938833, + "440577711/train2": 0.28744401244938833, + "441046872/train0": -0.07385947182024077, + "441046872/train1": -0.07385947182024077, + "441046872/train2": -0.07385947182024077, + "509430392/train0": 0.28744401244938833, + "509430392/train1": 0.28744401244938833, + "509430392/train2": 0.28744401244938833, + "585904130/train0": -0.0718611703384171, + "585904130/train1": -0.0718611703384171, + "585904130/train2": -0.0718611703384171, + "608016290/train0": -0.07385947182024077, + "608016290/train1": -0.07385947182024077, + "608016290/train2": -0.07385947182024077, + "626486813/train0": -0.0718611703384171, + "626486813/train1": -0.0718611703384171, + "626486813/train2": -0.0718611703384171, + "636680137/train0": -0.0718611703384171, + "636680137/train1": -0.0718611703384171, + "636680137/train2": -0.0718611703384171, + "655884605/train0": -0.18710182505305797, + "655884605/train1": -0.18710182505305797, + "655884605/train2": -0.18710182505305797, + "672328262/train0": -0.0718611703384171, + "672328262/train1": -0.0718611703384171, + "672328262/train2": -0.0718611703384171, + "701667486/train0": -0.18710182505305797, + "701667486/train1": -0.18710182505305797, + "701667486/train2": -0.18710182505305797, + "747260437/train0": -0.0718611703384171, + "747260437/train1": -0.0718611703384171, + "747260437/train2": -0.0718611703384171, + "750448296/train0": -0.0718611703384171, + "750448296/train1": -0.0718611703384171, + "750448296/train2": -0.0718611703384171, + "767747424/train0": -0.0718611703384171, + "767747424/train1": -0.0718611703384171, + "767747424/train2": -0.0718611703384171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.14292261874153667, + "low": -0.14140522346356643, + "mean": -0.0032829223622384716, + "n": 5, + "num_samples": 100 + }, + "mean": -0.0032829223622384716, + "per_task": { + "LiftPegUpright-v1": -0.07385947182024077, + "PickCube-v1": 0.02896384295113541, + "PullCube-v1": -0.07186117033841724, + "PushCube-v1": 0.2874440124493883, + "StackCube-v1": -0.18710182505305803 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": -0.02401517240035991, + "low": -0.07481897039354908, + "mean": -0.05146497448953768, + "n": 144, + "num_samples": 100 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": { + "LiftPegUpright-v1": -0.07385947182024077, + "PickCube-v1": 0.02896384295113541, + "PullCube-v1": -0.0718611703384171, + "PushCube-v1": 0.28744401244938833, + "StackCube-v1": -0.18710182505305797, + "__global__": -0.0718611703384171 + }, + "threshold_scope": "task", + "weights": [ + -0.0641907829395102, + 0.13421925424850267, + 0.051446255445118216, + -0.02692717312099696, + 0.03949937468823339, + 0.0031610982114552155, + -0.030533265050200487, + -0.2466518749150703, + 0.13858509929641644, + 0.44337844473353627, + -0.30014743551396617, + 0.028331214966580642, + -0.41892530344398793, + 0.1485314741027195, + 0.3636929023950852, + 0.008425611584092718, + -0.01249796859986918, + 9.026258089869788e-05, + -0.00830387725837248, + 0.14429534705589298, + -0.21947883728389073, + -0.3419846991571873, + 0.4503129094083618, + -0.31264769936132564, + -0.013503521549163455, + -2.4420171938464715e-15, + -0.07579997170543037, + -0.13622789692025705, + -0.007750667166434494 + ] +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..be07980886dd2def2fe9831e642b7e3108c27724 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,3236 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3865546218487395, + "pairwise_causal_calibration_confidence": 0.5251040975256128, + "pairwise_causal_calibration_ece": 0.13854947567687334, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.2551750656075202, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.67014479637146, + "selected_utility_gain_over_base": -1.32985520362854, + "selector_regret": 1.32985520362854, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3416666666666667, + "pairwise_causal_calibration_confidence": 0.5423238840429205, + "pairwise_causal_calibration_ece": 0.20065721737625392, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.20299343323495855, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6947963237762451, + "selected_utility_gain_over_base": -1.3052036762237549, + "selector_regret": 1.3052036762237549, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8809523809523809, + "pairwise_causal_calibration_confidence": 0.5368743958607693, + "pairwise_causal_calibration_ece": 0.3440779850916114, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.24613098923868526, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5379810554394029, + "pairwise_causal_calibration_ece": 0.06571145427295252, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.15799813692233977, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09047546982765198, + "selected_utility_gain_over_base": -0.06794795393943787, + "selector_regret": 0.012273147702217102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": -0.18710182505305797 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7142857142857143, + "pairwise_causal_calibration_confidence": 0.5252684472642476, + "pairwise_causal_calibration_ece": 0.1890172670214667, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.13463328766802296, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18093222379684448, + "selected_utility_gain_over_base": 0.02250880002975464, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": -0.18710182505305797 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5357142857142857, + "pairwise_causal_calibration_confidence": 0.5332614138108104, + "pairwise_causal_calibration_ece": 0.0024528719034753177, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.0627076273903946, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14433109760284424, + "selected_utility_gain_over_base": -0.014092326164245605, + "selector_regret": 0.05094057321548462, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": -0.18710182505305797 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4095238095238095, + "pairwise_causal_calibration_confidence": 0.51710870319415, + "pairwise_causal_calibration_ece": 0.10758489367034052, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.030788448510796762, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15656626224517822, + "selected_utility_gain_over_base": -0.06943503022193909, + "selector_regret": 0.7097858786582947, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": -0.18710182505305797 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.580952380952381, + "pairwise_causal_calibration_confidence": 0.5316022421578545, + "pairwise_causal_calibration_ece": 0.049350138794526544, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.05730648283766754, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8148510456085205, + "selected_utility_gain_over_base": 0.5888497531414032, + "selector_regret": 1.1851489543914795, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4666666666666667, + "pairwise_causal_calibration_confidence": 0.5177275439295085, + "pairwise_causal_calibration_ece": 0.05106087726284181, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.033176433226523416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18784314393997192, + "selected_utility_gain_over_base": -0.038158148527145386, + "selector_regret": 0.36436158418655396, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": -0.18710182505305797 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5050505050505051, + "pairwise_causal_calibration_confidence": 0.5386347584075308, + "pairwise_causal_calibration_ece": 0.03358425335702575, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.3447490352381858, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5909090909090909, + "pairwise_causal_calibration_confidence": 0.5422560685706671, + "pairwise_causal_calibration_ece": 0.048653022338423835, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.2330187222265443, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5952380952380952, + "pairwise_causal_calibration_confidence": 0.5426310160661976, + "pairwise_causal_calibration_ece": 0.05260707917189755, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.3117657280355065, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08501242101192474, + "selected_utility_gain_over_base": -1.9149875789880753, + "selector_regret": 1.9149875789880753, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.39285714285714285, + "pairwise_causal_calibration_confidence": 0.5333367283214293, + "pairwise_causal_calibration_ece": 0.14047958546428646, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1742924240977557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.028782829642295837, + "selected_utility_gain_over_base": -0.06342208385467529, + "selector_regret": 0.026206374168395996, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": -0.18710182505305797 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.35714285714285715, + "pairwise_causal_calibration_confidence": 0.5357950970847783, + "pairwise_causal_calibration_ece": 0.1786522399419212, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": 0.18792055489105874, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06454251706600189, + "selected_utility_gain_over_base": -0.02766239643096924, + "selector_regret": 0.067916139960289, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": -0.18710182505305797 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5266268320102542, + "pairwise_causal_calibration_ece": 0.026626832010254176, + "pairwise_causal_calibration_pairs": 28.0, + "predicted_margin": -0.1739828724101382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13575458526611328, + "selected_utility_gain_over_base": 0.04354967176914215, + "selector_regret": 0.05715706944465637, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": -0.18710182505305797 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5267509882327646, + "pairwise_causal_calibration_ece": 0.043413085733646106, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6155609171443972, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.054066285490989685, + "selected_utility_gain_over_base": -0.052404239773750305, + "selector_regret": 0.08999375998973846, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.28744401244938833 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.5232561219266944, + "pairwise_causal_calibration_ece": 0.035861163943501084, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.24134426910254947, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6181818181818182, + "pairwise_causal_calibration_confidence": 0.5209245623930986, + "pairwise_causal_calibration_ece": 0.09725725578871958, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": -0.06739236477010135, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49333333333333335, + "pairwise_causal_calibration_confidence": 0.5169520118772317, + "pairwise_causal_calibration_ece": 0.023618678543898375, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": -0.07560538396524885, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5789473684210527, + "pairwise_causal_calibration_confidence": 0.5298121533009607, + "pairwise_causal_calibration_ece": 0.049135215120092, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": -0.4256345336980503, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44047619047619047, + "pairwise_causal_calibration_confidence": 0.5313546391394341, + "pairwise_causal_calibration_ece": 0.09087844866324368, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.3658750831677004, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4583333333333333, + "pairwise_causal_calibration_confidence": 0.5279271325774133, + "pairwise_causal_calibration_ece": 0.06959379924407999, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.008737131332534287, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007268309593200684, + "selected_utility_gain_over_base": -0.09556593745946884, + "selector_regret": 0.007909595966339111, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": -0.0718611703384171 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.575, + "pairwise_causal_calibration_confidence": 0.5244535198032577, + "pairwise_causal_calibration_ece": 0.05054648019674224, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.11792927817531362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030119141563773155, + "selected_utility_gain_over_base": -0.07271510548889637, + "selector_regret": 0.006891986355185509, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": -0.0718611703384171 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5083333333333333, + "pairwise_causal_calibration_confidence": 0.538380386098592, + "pairwise_causal_calibration_ece": 0.043205574824282285, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.034121695817442343, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.058951716870069504, + "selected_utility_gain_over_base": -0.04388253018260002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": -0.0718611703384171 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49166666666666664, + "pairwise_causal_calibration_confidence": 0.5251639529546277, + "pairwise_causal_calibration_ece": 0.03349728628796106, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4675581452853047, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02898651361465454, + "selected_utility_gain_over_base": -0.06491655111312866, + "selector_regret": 0.03121185302734375, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": -0.0718611703384171 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5166360139443322, + "pairwise_causal_calibration_ece": 0.08330268061099888, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4500223738950829, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009051382541656494, + "selected_utility_gain_over_base": -0.08485168218612671, + "selector_regret": 0.007533630356192589, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": -0.0718611703384171 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.519839782973733, + "pairwise_causal_calibration_ece": 0.08016021702626697, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.4462633698719307, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06987287849187851, + "selected_utility_gain_over_base": -0.024030186235904694, + "selector_regret": 0.03010360151529312, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": -0.0718611703384171 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.41414141414141414, + "pairwise_causal_calibration_confidence": 0.5226475437853878, + "pairwise_causal_calibration_ece": 0.10850612964397366, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.6476723025740536, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07608599960803986, + "selected_utility_gain_over_base": -0.010948076844215393, + "selector_regret": 1.9239140003919601, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48739495798319327, + "pairwise_causal_calibration_confidence": 0.5136404664974943, + "pairwise_causal_calibration_ece": 0.02624550851430102, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.38625575833343895, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19651852548122406, + "selected_utility_gain_over_base": 0.10948444902896881, + "selector_regret": 1.803481474518776, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5882352941176471, + "pairwise_causal_calibration_confidence": 0.5148540856771467, + "pairwise_causal_calibration_ece": 0.07338120844050033, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.35255742400707646, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21615806221961975, + "selected_utility_gain_over_base": 0.1291239857673645, + "selector_regret": 1.7838419377803802, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43333333333333335, + "pairwise_causal_calibration_confidence": 0.5158281275240727, + "pairwise_causal_calibration_ece": 0.08249479419073935, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.12116588314285559, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.30158730158730157, + "pairwise_causal_calibration_confidence": 0.5242300284642385, + "pairwise_causal_calibration_ece": 0.2226427268769369, + "pairwise_causal_calibration_pairs": 63.0, + "predicted_margin": -0.10941040668136354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8129013180732727, + "selected_utility_gain_over_base": -0.02050870656967163, + "selector_regret": 1.1870986819267273, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5111111111111111, + "pairwise_causal_calibration_confidence": 0.5240930381172986, + "pairwise_causal_calibration_ece": 0.012981927006187598, + "pairwise_causal_calibration_pairs": 90.0, + "predicted_margin": -0.14659151504624657, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16154873371124268, + "selected_utility_gain_over_base": -0.6718612909317017, + "selector_regret": 1.8384512662887573, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6944444444444444, + "pairwise_causal_calibration_confidence": 0.5457014445746324, + "pairwise_causal_calibration_ece": 0.14874299986981188, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.23967836808523524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.02896384295113541 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5555555555555556, + "pairwise_causal_calibration_confidence": 0.5111398905599821, + "pairwise_causal_calibration_ece": 0.04441566499557348, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.2514915262180857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.02896384295113541 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3611111111111111, + "pairwise_causal_calibration_confidence": 0.525252740289532, + "pairwise_causal_calibration_ece": 0.16414162917842084, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.28676975669762994, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.02896384295113541 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.75, + "pairwise_causal_calibration_confidence": 0.5337482570523958, + "pairwise_causal_calibration_ece": 0.21625174294760421, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.5713459023104054, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03462250903248787, + "selected_utility_gain_over_base": -0.06989232823252678, + "selector_regret": 0.026248913258314133, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": -0.0718611703384171 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.375, + "pairwise_causal_calibration_confidence": 0.5262268456146021, + "pairwise_causal_calibration_ece": 0.1512268456146021, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.3107421704514528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": -0.0718611703384171 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.25833333333333336, + "pairwise_causal_calibration_confidence": 0.528533525486533, + "pairwise_causal_calibration_ece": 0.2702001921531997, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.6769572594134817, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04690708592534065, + "selected_utility_gain_over_base": -0.057607751339673996, + "selector_regret": 0.06619475409388542, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": -0.0718611703384171 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5470085470085471, + "pairwise_causal_calibration_confidence": 0.5361003534463802, + "pairwise_causal_calibration_ece": 0.042038781082623024, + "pairwise_causal_calibration_pairs": 117.0, + "predicted_margin": -0.11714129730488587, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14173981547355652, + "selected_utility_gain_over_base": -0.6823966801166534, + "selector_regret": 1.8582601845264435, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5454545454545454, + "pairwise_causal_calibration_confidence": 0.5394992318577098, + "pairwise_causal_calibration_ece": 0.038926080840327754, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": -0.09068349284851132, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3949579831932773, + "pairwise_causal_calibration_confidence": 0.5340698575932721, + "pairwise_causal_calibration_ece": 0.1391118743999947, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.14154551725382616, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1460946798324585, + "selected_utility_gain_over_base": -0.6780418157577515, + "selector_regret": 1.8539053201675415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.3567590128192366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.19263300195342614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "pairwise_causal_calibration_pairs": 0.0, + "predicted_margin": 0.2551836974507351, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6272727272727273, + "pairwise_causal_calibration_confidence": 0.5316179775586928, + "pairwise_causal_calibration_ece": 0.0956547497140344, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.5329093436927219, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.543859649122807, + "pairwise_causal_calibration_confidence": 0.5328964743528634, + "pairwise_causal_calibration_ece": 0.010963174769943684, + "pairwise_causal_calibration_pairs": 114.0, + "predicted_margin": 0.5122492091762928, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.43636363636363634, + "pairwise_causal_calibration_confidence": 0.5352532151194661, + "pairwise_causal_calibration_ece": 0.0988895787558297, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.4672762110936981, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13743820786476135, + "selected_utility_gain_over_base": 0.012670740485191345, + "selector_regret": 1.8625617921352386, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.525, + "pairwise_causal_calibration_confidence": 0.539953848754645, + "pairwise_causal_calibration_ece": 0.15111259204903227, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.05440518485836532, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9280846938490868, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5415632384984795, + "pairwise_causal_calibration_ece": 0.10876536091501116, + "pairwise_causal_calibration_pairs": 49.0, + "predicted_margin": -0.03958497742915945, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9280846938490868, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5237701807590434, + "pairwise_causal_calibration_ece": 0.026229819240956687, + "pairwise_causal_calibration_pairs": 40.0, + "predicted_margin": -0.16858629361831445, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2608695652173913, + "pairwise_causal_calibration_confidence": 0.5171237518534777, + "pairwise_causal_calibration_ece": 0.2562541866360864, + "pairwise_causal_calibration_pairs": 46.0, + "predicted_margin": -0.13774783214464598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38461538461538464, + "pairwise_causal_calibration_confidence": 0.5121094164593272, + "pairwise_causal_calibration_ece": 0.12749403184394253, + "pairwise_causal_calibration_pairs": 13.0, + "predicted_margin": -0.07780183911371158, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.52, + "pairwise_causal_calibration_confidence": 0.5170849899671911, + "pairwise_causal_calibration_ece": 0.002915010032808918, + "pairwise_causal_calibration_pairs": 25.0, + "predicted_margin": -0.12444815357019567, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.2571428571428571, + "pairwise_causal_calibration_confidence": 0.5267116008364124, + "pairwise_causal_calibration_ece": 0.2695687436935554, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": -0.11408015116078918, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14746195077896118, + "selected_utility_gain_over_base": -1.8525380492210388, + "selector_regret": 1.8525380492210388, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44285714285714284, + "pairwise_causal_calibration_confidence": 0.5323449915161098, + "pairwise_causal_calibration_ece": 0.10072366188957278, + "pairwise_causal_calibration_pairs": 70.0, + "predicted_margin": -0.17466576670619666, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5657894736842105, + "pairwise_causal_calibration_confidence": 0.5344673895535903, + "pairwise_causal_calibration_ece": 0.031322084130620015, + "pairwise_causal_calibration_pairs": 76.0, + "predicted_margin": -0.16215465004219584, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8302689790725708, + "selected_utility_gain_over_base": -1.1697310209274292, + "selector_regret": 1.1697310209274292, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.3025210084033613, + "pairwise_causal_calibration_confidence": 0.5305966527751617, + "pairwise_causal_calibration_ece": 0.22807564437180022, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.2180007600904328, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7026570439338684, + "selected_utility_gain_over_base": -0.020948290824890137, + "selector_regret": 1.2973429560661316, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.425, + "pairwise_causal_calibration_confidence": 0.5238503061013462, + "pairwise_causal_calibration_ece": 0.09885030610134621, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.16726605949103523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6900332570075989, + "selected_utility_gain_over_base": -0.03357207775115967, + "selector_regret": 1.3099667429924011, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5, + "pairwise_causal_calibration_confidence": 0.5242257655342142, + "pairwise_causal_calibration_ece": 0.024225765534214183, + "pairwise_causal_calibration_pairs": 92.0, + "predicted_margin": 0.24853085052742357, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6829424500465393, + "selected_utility_gain_over_base": -0.04066288471221924, + "selector_regret": 1.3170575499534607, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.0, + "pairwise_causal_calibration_confidence": 0.5768622164838934, + "pairwise_causal_calibration_ece": 0.5768622164838934, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.08574322718445004, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04285776615142822, + "selected_utility_gain_over_base": -1.9571422338485718, + "selector_regret": 1.9571422338485718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5740740740740741, + "pairwise_causal_calibration_confidence": 0.5594515320080404, + "pairwise_causal_calibration_ece": 0.2553362522454674, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.1354107312118617, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5151515151515151, + "pairwise_causal_calibration_confidence": 0.5363991279304782, + "pairwise_causal_calibration_ece": 0.03713806757575985, + "pairwise_causal_calibration_pairs": 99.0, + "predicted_margin": 0.08745956040002545, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12977690994739532, + "selected_utility_gain_over_base": -1.8702230900526047, + "selector_regret": 1.8702230900526047, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45, + "pairwise_causal_calibration_confidence": 0.5138396482606322, + "pairwise_causal_calibration_ece": 0.031919824130316105, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": 0.2227645387030986, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 1.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0391597636044025, + "selected_utility_gain_over_base": -0.9608402363955975, + "selector_regret": 0.03812338784337044, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": -0.0718611703384171 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.49122807017543857, + "pairwise_causal_calibration_confidence": 0.5227936475539869, + "pairwise_causal_calibration_ece": 0.015782788689274185, + "pairwise_causal_calibration_pairs": 57.0, + "predicted_margin": 0.2770881776119133, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0660420358181, + "selected_utility_gain_over_base": -0.9339579641819, + "selector_regret": 0.9339579641819, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.4369747899159664, + "pairwise_causal_calibration_confidence": 0.5252379082880491, + "pairwise_causal_calibration_ece": 0.044131559186041336, + "pairwise_causal_calibration_pairs": 59.5, + "predicted_margin": 0.1326005713312028, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0276100430637598, + "selected_utility_gain_over_base": -0.9723899569362402, + "selector_regret": 0.9723899569362402, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6388888888888888, + "pairwise_causal_calibration_confidence": 0.5184746997192393, + "pairwise_causal_calibration_ece": 0.12041418916964952, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.13802527783024798, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.043751150369644165, + "selected_utility_gain_over_base": -0.008499354124069214, + "selector_regret": 0.054248154163360596, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": -0.18710182505305797 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4722222222222222, + "pairwise_causal_calibration_confidence": 0.5119131115242276, + "pairwise_causal_calibration_ece": 0.03969088930200537, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.0925196999812683, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07580336928367615, + "selected_utility_gain_over_base": 0.02355286478996277, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": -0.18710182505305797 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8055555555555556, + "pairwise_causal_calibration_confidence": 0.5138245455754116, + "pairwise_causal_calibration_ece": 0.291731009980144, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.061796487673025036, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11105132102966309, + "selected_utility_gain_over_base": 0.05880081653594971, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": -0.18710182505305797 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7058823529411765, + "pairwise_causal_calibration_confidence": 0.5351161430732934, + "pairwise_causal_calibration_ece": 0.17076620986788324, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.16494316577956913, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12586313486099243, + "selected_utility_gain_over_base": -0.06504528224468231, + "selector_regret": 1.8741368651390076, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8416666666666667, + "pairwise_causal_calibration_confidence": 0.5382974229033135, + "pairwise_causal_calibration_ece": 0.30336924376335306, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.10202293411527177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": -0.18710182505305797 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5317298952114761, + "pairwise_causal_calibration_ece": 0.06827010478852391, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.26370842901349006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04297435283660889, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": -0.18710182505305797 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5568181818181819, + "pairwise_causal_calibration_confidence": 0.519833289537298, + "pairwise_causal_calibration_ece": 0.13700175449974886, + "pairwise_causal_calibration_pairs": 109.5, + "predicted_margin": 0.1024719525224913, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06754684634506702, + "selected_utility_gain_over_base": 0.02481185644865036, + "selector_regret": 1.0058094765990973, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": -0.0718611703384171 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5907608695652173, + "pairwise_causal_calibration_confidence": 0.5198421814375511, + "pairwise_causal_calibration_ece": 0.1186012755305571, + "pairwise_causal_calibration_pairs": 106.0, + "predicted_margin": 0.16388738649092643, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 5.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0095977680757642, + "selected_utility_gain_over_base": 0.9668627781793475, + "selector_regret": 0.028076826594769955, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": -0.0718611703384171 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.33611111111111114, + "pairwise_causal_calibration_confidence": 0.5255283835156899, + "pairwise_causal_calibration_ece": 0.1894172724045787, + "pairwise_causal_calibration_pairs": 109.5, + "predicted_margin": 0.16857295234438457, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.034129787236452, + "selected_utility_gain_over_base": 0.9913947973400354, + "selector_regret": 0.10181912407279015, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.42857142857142855, + "pairwise_causal_calibration_confidence": 0.5343458881100447, + "pairwise_causal_calibration_ece": 0.10577445953861614, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.3084681084329295, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": -0.07385947182024077 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.46153846153846156, + "pairwise_causal_calibration_confidence": 0.5333254751964351, + "pairwise_causal_calibration_ece": 0.07178701365797358, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.22321666120891376, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": -0.07385947182024077 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.3626373626373626, + "pairwise_causal_calibration_confidence": 0.5472048889164063, + "pairwise_causal_calibration_ece": 0.1845675262790436, + "pairwise_causal_calibration_pairs": 91.0, + "predicted_margin": -0.04845957189152705, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20841218531131744, + "selected_utility_gain_over_base": -0.3916783779859543, + "selector_regret": 0.021801263093948364, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": -0.07385947182024077 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.35, + "pairwise_causal_calibration_confidence": 0.5258131375323639, + "pairwise_causal_calibration_ece": 0.17581313753236394, + "pairwise_causal_calibration_pairs": 60.0, + "predicted_margin": -0.10215255174920451, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.14946338534355164, + "selected_utility_gain_over_base": -1.8505366146564484, + "selector_regret": 1.8505366146564484, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38461538461538464, + "pairwise_causal_calibration_confidence": 0.523423235146417, + "pairwise_causal_calibration_ece": 0.13880785053103237, + "pairwise_causal_calibration_pairs": 39.0, + "predicted_margin": -0.15959785934997645, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125194311141968, + "selected_utility_gain_over_base": -1.1874805688858032, + "selector_regret": 1.1874805688858032, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6309523809523809, + "pairwise_causal_calibration_confidence": 0.5248002449882088, + "pairwise_causal_calibration_ece": 0.12044288641048877, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.20549642192166445, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5862068965517241, + "pairwise_causal_calibration_confidence": 0.5272168394784098, + "pairwise_causal_calibration_ece": 0.058990057073314284, + "pairwise_causal_calibration_pairs": 29.0, + "predicted_margin": 0.7070441087351717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.8666666666666667, + "pairwise_causal_calibration_confidence": 0.5379425484601451, + "pairwise_causal_calibration_ece": 0.3287241182065216, + "pairwise_causal_calibration_pairs": 15.0, + "predicted_margin": 0.7017571830152312, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.7222222222222222, + "pairwise_causal_calibration_confidence": 0.536464978436527, + "pairwise_causal_calibration_ece": 0.18575724378569516, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": 0.6922087959324436, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.38181818181818183, + "pairwise_causal_calibration_confidence": 0.5381289217490534, + "pairwise_causal_calibration_ece": 0.15631073993087152, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": 0.009282072163031563, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20389758050441742, + "selected_utility_gain_over_base": -0.49605758488178253, + "selector_regret": 1.7961024194955826, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.509090909090909, + "pairwise_causal_calibration_confidence": 0.5333064538046144, + "pairwise_causal_calibration_ece": 0.02421554471370535, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.11963786997800063, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": -0.07385947182024077 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5636363636363636, + "pairwise_causal_calibration_confidence": 0.5375543539918383, + "pairwise_causal_calibration_ece": 0.03382332741347877, + "pairwise_causal_calibration_pairs": 55.0, + "predicted_margin": -0.08224411989102232, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": -0.07385947182024077 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5378151260504201, + "pairwise_causal_calibration_confidence": 0.5162619682864167, + "pairwise_causal_calibration_ece": 0.02155315776400346, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.5136988024654346, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5280283691474993, + "pairwise_causal_calibration_ece": 0.04340020228107211, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.4261006779620051, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.28744401244938833 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6302521008403361, + "pairwise_causal_calibration_confidence": 0.5305902079798236, + "pairwise_causal_calibration_ece": 0.09966189286051252, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": 0.3587989801510558, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 5.424022674560547e-06, + "selected_utility_gain_over_base": 5.200505256652832e-06, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.28744401244938833 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.44047619047619047, + "pairwise_causal_calibration_confidence": 0.5212774691287471, + "pairwise_causal_calibration_ece": 0.0808012786525566, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": 0.08651310891179698, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4533333333333333, + "pairwise_causal_calibration_confidence": 0.5302663446880443, + "pairwise_causal_calibration_ece": 0.076933011354711, + "pairwise_causal_calibration_pairs": 75.0, + "predicted_margin": 0.09881997958173372, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6818181818181818, + "pairwise_causal_calibration_confidence": 0.522915616847175, + "pairwise_causal_calibration_ece": 0.15890256497100674, + "pairwise_causal_calibration_pairs": 110.0, + "predicted_margin": 0.0589756379475741, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08930286020040512, + "selected_utility_gain_over_base": -0.09241268783807755, + "selector_regret": 1.9106971397995949, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5904761904761905, + "pairwise_causal_calibration_confidence": 0.5293645053307068, + "pairwise_causal_calibration_ece": 0.14243134895449866, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": 0.08072228079745068, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5357192536556177, + "pairwise_causal_calibration_ece": 0.0682003715539854, + "pairwise_causal_calibration_pairs": 105.0, + "predicted_margin": -0.1274552507433626, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.07385947182024077 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.44166666666666665, + "pairwise_causal_calibration_confidence": 0.5315196569218078, + "pairwise_causal_calibration_ece": 0.08985299025514107, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.030776181488394214, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21245408058166504, + "selected_utility_gain_over_base": -0.513247549533844, + "selector_regret": 0.49927353858947754, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": -0.07385947182024077 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.6125, + "pairwise_causal_calibration_confidence": 0.5273776063219382, + "pairwise_causal_calibration_ece": 0.08512239367806185, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.34770303375956, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 6.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.011467556469142437, + "selected_utility_gain_over_base": 0.006369213748257607, + "selector_regret": 0.0332279191352427, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": -0.0718611703384171 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.48333333333333334, + "pairwise_causal_calibration_confidence": 0.521404963299847, + "pairwise_causal_calibration_ece": 0.03807162996651359, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.2450842010496616, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015682418830692768, + "selected_utility_gain_over_base": 0.010584076109807938, + "selector_regret": 0.020145455840975046, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": -0.0718611703384171 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.531833628670964, + "pairwise_causal_calibration_ece": 0.056833628670964004, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.22550405863537087, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 9.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030346781481057405, + "selected_utility_gain_over_base": 0.025248438760172576, + "selector_regret": 0.014552473556250334, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": -0.0718611703384171 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.5333333333333333, + "pairwise_causal_calibration_confidence": 0.5395580589892535, + "pairwise_causal_calibration_ece": 0.006224725655920161, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.8489010665175709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08826130628585815, + "selected_utility_gain_over_base": 0.005126260221004486, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": -0.0718611703384171 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.55, + "pairwise_causal_calibration_confidence": 0.5207010539912516, + "pairwise_causal_calibration_ece": 0.029298946008748494, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.39916002861669686, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017789265140891075, + "selected_utility_gain_over_base": -0.0653457809239626, + "selector_regret": 0.04962191917002201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": -0.0718611703384171 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5583333333333333, + "pairwise_causal_calibration_confidence": 0.5159516842599035, + "pairwise_causal_calibration_ece": 0.042381649073429895, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.47473165550706736, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03889387845993042, + "selected_utility_gain_over_base": -0.04424116760492325, + "selector_regret": 0.03675774484872818, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": -0.0718611703384171 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6388888888888888, + "pairwise_causal_calibration_confidence": 0.5310489483070069, + "pairwise_causal_calibration_ece": 0.10783994058188195, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.09399166521150604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12222731113433838, + "selected_utility_gain_over_base": -0.040779367089271545, + "selector_regret": 0.015101373195648193, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": -0.18710182505305797 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4444444444444444, + "pairwise_causal_calibration_confidence": 0.5070899546645378, + "pairwise_causal_calibration_ece": 0.06264551022009335, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.07407655216726083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12105989456176758, + "selected_utility_gain_over_base": -0.041946783661842346, + "selector_regret": 0.6918159127235413, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": -0.18710182505305797 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4166666666666667, + "pairwise_causal_calibration_confidence": 0.5234811460445998, + "pairwise_causal_calibration_ece": 0.10681447937793315, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": 0.07294104079981856, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21447330713272095, + "selected_utility_gain_over_base": 0.05146662890911102, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": -0.18710182505305797 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5294117647058824, + "pairwise_causal_calibration_confidence": 0.5321127842961938, + "pairwise_causal_calibration_ece": 0.0027010195903114376, + "pairwise_causal_calibration_pairs": 17.0, + "predicted_margin": -0.11406027153604881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6, + "pairwise_causal_calibration_confidence": 0.5540392504779817, + "pairwise_causal_calibration_ece": 0.06500635112191766, + "pairwise_causal_calibration_pairs": 35.0, + "predicted_margin": -0.16708550898936614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.13333333333333333, + "pairwise_causal_calibration_confidence": 0.5470577940231715, + "pairwise_causal_calibration_ece": 0.41372446068983815, + "pairwise_causal_calibration_pairs": 30.0, + "predicted_margin": -0.1772708238485603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.0718611703384171 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5714285714285714, + "pairwise_causal_calibration_confidence": 0.5385441284405091, + "pairwise_causal_calibration_ece": 0.06175922923362397, + "pairwise_causal_calibration_pairs": 42.0, + "predicted_margin": -0.0742733989521003, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.6666666666666666, + "pairwise_causal_calibration_confidence": 0.5308940956422026, + "pairwise_causal_calibration_ece": 0.1357725710244641, + "pairwise_causal_calibration_pairs": 84.0, + "predicted_margin": -0.061055849565622045, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.8888888888888888, + "pairwise_causal_calibration_confidence": 0.5566493339472486, + "pairwise_causal_calibration_ece": 0.3322395549416404, + "pairwise_causal_calibration_pairs": 54.0, + "predicted_margin": -0.00871862718880162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": -0.18710182505305797 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.38333333333333336, + "pairwise_causal_calibration_confidence": 0.522326045653055, + "pairwise_causal_calibration_ece": 0.1389927123197216, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.04417027513292293, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04841095209121704, + "selected_utility_gain_over_base": -1.951589047908783, + "selector_regret": 0.059984683990478516, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": -0.0718611703384171 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.475, + "pairwise_causal_calibration_confidence": 0.5240130464861401, + "pairwise_causal_calibration_ece": 0.04901304648614013, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": -0.15678779044109292, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": -0.0718611703384171 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.45, + "pairwise_causal_calibration_confidence": 0.5277135352836209, + "pairwise_causal_calibration_ece": 0.07771353528362085, + "pairwise_causal_calibration_pairs": 120.0, + "predicted_margin": 0.08370080144449699, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06004788354039192, + "selected_utility_gain_over_base": -1.939952116459608, + "selector_regret": 0.06505399569869041, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": -0.0718611703384171 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5869047619047619, + "pairwise_causal_calibration_confidence": 0.5367471317580962, + "pairwise_causal_calibration_ece": 0.07635409425053818, + "pairwise_causal_calibration_pairs": 108.0, + "predicted_margin": -0.04826188693738287, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08246687302986781, + "selected_utility_gain_over_base": -0.035126276314258575, + "selector_regret": 0.670193454871575, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": -0.0718611703384171 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "pairwise_causal_calibration_accuracy": 0.4262820512820513, + "pairwise_causal_calibration_confidence": 0.5306744409251447, + "pairwise_causal_calibration_ece": 0.13910389780114304, + "pairwise_causal_calibration_pairs": 101.66666666666667, + "predicted_margin": -0.20270377858890484, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 11.333333333333334, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10720233743389447, + "selected_utility_gain_over_base": -0.010390811910231909, + "selector_regret": 1.2633009366691113, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": -0.0718611703384171 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "pairwise_causal_calibration_accuracy": 0.5674603174603174, + "pairwise_causal_calibration_confidence": 0.5335771984593171, + "pairwise_causal_calibration_ece": 0.14091077521706144, + "pairwise_causal_calibration_pairs": 108.0, + "predicted_margin": -0.10917174959959015, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11559025570750237, + "selected_utility_gain_over_base": -0.002002893636624018, + "selector_regret": 1.23485841229558, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": -0.0718611703384171 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5088675213675213, + "pairwise_causal_calibration_confidence": 0.5216055060310809, + "pairwise_causal_calibration_ece": 0.08288086769862717, + "pairwise_causal_calibration_pairs": 119.25, + "predicted_margin": -0.06789015981965127, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 5.5, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5665738391689956, + "selected_utility_gain_over_base": -0.006625502835959196, + "selector_regret": 0.4758553486317396, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": -0.0718611703384171 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.75, + "execute_generated": 0.75, + "fallback_rate": 0.25, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.43937280040221216, + "pairwise_causal_calibration_confidence": 0.5291623924295586, + "pairwise_causal_calibration_ece": 0.08978959202734649, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.004134822890086619, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 6.5, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.07074803306022659, + "selected_utility_gain_over_base": -0.5024513089447282, + "selector_regret": 0.9922050581662916, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": -0.0718611703384171 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "pairwise_causal_calibration_accuracy": 0.5861649788120377, + "pairwise_causal_calibration_confidence": 0.5228149250617854, + "pairwise_causal_calibration_ece": 0.0822416726337075, + "pairwise_causal_calibration_pairs": 119.0, + "predicted_margin": -0.07224136268104218, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 9.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6022169017232955, + "selected_utility_gain_over_base": 0.029017559718340635, + "selector_regret": 0.46495019318535924, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": -0.0718611703384171 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..ed762abf2f6f967d6a691f74a19fec15bc4aa41e --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,137 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "pairwise_causal_calibration_accuracy": 0.4784086828204475, + "pairwise_causal_calibration_confidence": 0.5335629666344366, + "pairwise_causal_calibration_ece": 0.1274266477632942, + "pairwise_causal_calibration_pairs": 94.8, + "predicted_margin": 0.03252291650109252, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 7.333333333333333, + "selected_success": 0.13333333333333333, + "selected_success_gain_over_base": -0.06666666666666667, + "selected_utility": 0.7594087203343709, + "selected_utility_gain_over_base": -0.19046181837717693, + "selector_regret": 0.6795656750599544, + "success_selector_gap": 0.4666666666666667, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": -0.07385947182024077 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "pairwise_causal_calibration_accuracy": 0.5370370370370371, + "pairwise_causal_calibration_confidence": 0.5273646918080489, + "pairwise_causal_calibration_ece": 0.11910009801460207, + "pairwise_causal_calibration_pairs": 36.0, + "predicted_margin": -0.25931321700031695, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 3.6666666666666665, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.02896384295113541 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.7333333333333333, + "execute_generated": 0.7333333333333333, + "fallback_rate": 0.26666666666666666, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "pairwise_causal_calibration_accuracy": 0.49303016307826086, + "pairwise_causal_calibration_confidence": 0.5282915670267949, + "pairwise_causal_calibration_ece": 0.09823916112294553, + "pairwise_causal_calibration_pairs": 92.32, + "predicted_margin": 0.0830997467005694, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 6.16, + "selected_success": 0.29333333333333333, + "selected_success_gain_over_base": -0.06666666666666667, + "selected_utility": 0.6347246278480937, + "selected_utility_gain_over_base": -0.14044583337536704, + "selector_regret": 0.5004223387533178, + "success_selector_gap": 0.25333333333333335, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": -0.07186117033841724 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.8333333333333334, + "execute_generated": 0.8333333333333334, + "fallback_rate": 0.16666666666666666, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "pairwise_causal_calibration_accuracy": 0.5754796516735593, + "pairwise_causal_calibration_confidence": 0.528992677243157, + "pairwise_causal_calibration_ece": 0.08128349345238421, + "pairwise_causal_calibration_pairs": 97.44444444444444, + "predicted_margin": 0.44319861689531703, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 6.944444444444445, + "selected_success": 0.4444444444444444, + "selected_success_gain_over_base": 0.2777777777777778, + "selected_utility": 0.9508692075808843, + "selected_utility_gain_over_base": 0.5584650288025538, + "selector_regret": 0.8349122073915269, + "success_selector_gap": 0.4444444444444444, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.2874440124493883 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.9393939393939394, + "execute_generated": 0.9393939393939394, + "fallback_rate": 0.06060606060606061, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "pairwise_causal_calibration_accuracy": 0.5307015979194418, + "pairwise_causal_calibration_confidence": 0.5283985674194418, + "pairwise_causal_calibration_ece": 0.12082192852053648, + "pairwise_causal_calibration_pairs": 67.45454545454545, + "predicted_margin": -0.05568968191367572, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 6.424242424242424, + "selected_success": 0.21212121212121213, + "selected_success_gain_over_base": -0.06060606060606061, + "selected_utility": 0.6115993509689966, + "selected_utility_gain_over_base": -0.16479535355712427, + "selector_regret": 0.5451958197535891, + "success_selector_gap": 0.30303030303030304, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": -0.18710182505305803 + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..92645a737339b3d6d513c05d6cd8ac2c026f4062 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_learned_dominance_selector.py +\begin{tabular}{lrrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\ +\midrule +144 & 0.7708 & 0.2292 & 0.2917 & 0.2708 & 0.5694 & 0.5486 & 0.2014 & 0.3056 & 0.0151 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b3e31e18ced6412c3f53f4f7907336088e4b6ce6 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_bundle_smoke_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained ridge dominance calibrator on calibration measured rows only +fit_objective=pointwise +selected_lambda=0.0 +tau={"LiftPegUpright-v1": -0.073859, "PickCube-v1": 0.028964, "PullCube-v1": -0.071861, "PushCube-v1": 0.287444, "StackCube-v1": -0.187102, "__global__": -0.071861} +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..3c80756af9c84e9bb9d5946fead21d2e182698df --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test --feature-set chart_bundle_consensus --selector-chart-feature-mode base_context_obs --target utility_margin --threshold-scope task --fit-objective pointwise --k 16 --bootstrap-samples 1 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7998e67d810eaebdceafc6d1613e0be654709aca --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 1 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_bundle_consensus +fit_objective: pointwise +k: 16 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs +source_index: None +success_bonus: 1.0 +target: utility_margin +threshold_scope: task diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_bundle_explore_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..5ca3127662ebf4427f4c86cdfbf04e702c83aa05 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_learned_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test --feature-set chart_source_bundle_consensus --selector-chart-feature-mode base_context_obs --target success_weighted_margin --success-bonus 2.0 --threshold-scope task --fit-objective pointwise --k 16 --bootstrap-samples 1 --no-markdown-report diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b82f440b2a612e7caaa882c384487739ae0bd351 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,18 @@ +bootstrap_samples: 1 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_source_bundle_consensus +fit_objective: pointwise +k: 16 +no_markdown_report: True +out_dir: runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test +pairwise_weight: 1.0 +ridge_lambdas: 0,0.01,0.1,1,10,100 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +success_bonus: 2.0 +target: success_weighted_margin +threshold_scope: task diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_learned_dominance_chart_source_bundle_explore_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +}