diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..1551166cd6f2b4be9215e175e7266f85b04075f0 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test --feature-set chart_source_compat --selector-chart-feature-mode base_context_obs --target utility_margin --k 16 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb95ae2274cfd1fe1f52ffdd4b234aa31621d8ed --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,15 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_source_compat +k: 16 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test +seed: 0 +selection_frac: 0.35 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +target: utility_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c001552e076e38f762c1980a9ac2fdd590af6dc4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ad0bf2d687219a6721c800177112229bd460a252 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,7488 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.5277777777777778, + "fallback_rate": 0.4722222222222222, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.2638888888888889, + "selected_success_gain_over_base": -0.027777777777777776, + "selected_utility": 0.6950111038766207, + "selector_regret": 0.5738296886734638, + "success_selector_gap": 0.3263888888888889, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "source_chart_found", + "source_positive_count_log", + "source_negative_count_log", + "source_nonbase_count_log", + "source_positive_rate", + "source_best_delta_utility", + "source_mean_delta_utility", + "source_best_utility", + "source_mean_utility", + "source_best_success", + "source_mean_success", + "source_best_progress", + "source_mean_progress", + "source_positive_safety_known_rate", + "source_positive_unsafe_rate_known", + "generated_to_source_positive_min_rms", + "generated_to_source_negative_min_rms", + "generated_source_pos_closer_than_neg", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "chart_source_compat", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4981949458483754, + "fallback_rate": 0.5018050541516246, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.5342960288808665, + "proposal_oracle_success": 0.5342960288808665, + "proposal_oracle_utility": 1.1336619752116286, + "selected_success": 0.4657039711191336, + "selected_success_gain_over_base": 0.18772563176895307, + "selected_utility": 1.0309938721758876, + "selector_regret": 0.21537994314825654, + "success_selector_gap": 0.11913357400722022, + "success_support_gap": 0.17328519855595667, + "support_gap": 0.3521794904517649 + }, + "k": 16, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.6129032258064516, + "fallback_rate": 0.3870967741935484, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.4064516129032258, + "selected_success_gain_over_base": 0.12903225806451613, + "selected_utility": 0.8838888318235336, + "selector_regret": 0.34334768112874026, + "success_selector_gap": 0.16774193548387098, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "num_fit_candidates": 4175, + "num_fit_rows": 277, + "num_selection_candidates": 2335, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17828726626426614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28418063141808364, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.34177985507477204, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.67014479637146, + "selected_utility_gain_over_base": -1.32985520362854, + "selector_regret": 1.32985520362854, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1565404769152266, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.33329724085783513, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6693241000175476, + "selected_utility_gain_over_base": -0.030631065368652344, + "selector_regret": 1.3306758999824524, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0722437728966473, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5849835367136861, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4633482246244871, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.029231786727905273, + "selected_utility_gain_over_base": -0.13245414197444916, + "selector_regret": 1.9707682132720947, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3547798191536652, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06019836664199829, + "selected_utility_gain_over_base": -0.03370469808578491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.8367442777727344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0025064547080546618, + "selected_utility_gain_over_base": 0.0005769530544057488, + "selector_regret": 0.007676919689401984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7606711355823724, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1048971638083458, + "selected_utility_gain_over_base": -1.8951028361916542, + "selector_regret": 0.049669139087200165, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04820260893743538, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.22231122454707192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.5081302857566511, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04563472792506218, + "selected_utility_gain_over_base": 0.009748421609401703, + "selector_regret": 0.10107791796326637, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24672540504885201, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.2761027146951176, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.06883030785742685, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3083190727440344, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05923032760620117, + "selected_utility_gain_over_base": -0.1224852204322815, + "selector_regret": 1.9407696723937988, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3220294334917424, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006893515586853027, + "selected_utility_gain_over_base": -0.0959407314658165, + "selector_regret": 0.008284389972686768, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15304893743210182, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.40225098671598103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.4338145851406176, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.011697530746459961, + "selected_utility_gain_over_base": -0.04242541268467903, + "selector_regret": 0.007504304870963097, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11744128190863853, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 12, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3359117235345434, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.8424992900631808, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.041394393891096115, + "selected_utility_gain_over_base": -0.05160309746861458, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8424422944342752, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.1675764505623386, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.47152384933302266, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07767494767904282, + "selected_utility_gain_over_base": 0.028091274201869965, + "selector_regret": 1.9223250523209572, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.7033632660299286, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.049984085020529354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14087048061856924, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18739562764215434, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4193019968970795, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14406004548072815, + "selected_utility_gain_over_base": 0.03758952021598816, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7041875516262702, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38402002330650076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5951794014623365, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3427743477646694, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06508849561214447, + "selected_utility_gain_over_base": -0.02194558084011078, + "selector_regret": 1.9349115043878555, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13472633878966278, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.26463671358858426, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2414028495277438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2150808050278072, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6195026651718122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8125001788139343, + "selected_utility_gain_over_base": -0.02090984582901001, + "selector_regret": 1.1874998211860657, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3392426447136978, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8090915828943253, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19860467482965133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15883484058846561, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5039098400899894, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1966920683523119, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 13, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2755208385556546, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20414937358844587, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2759242006849786, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4119603552284239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6916947960853577, + "selected_utility_gain_over_base": -1.3083052039146423, + "selector_regret": 1.3083052039146423, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1646347051814787, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3957300144448556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.663180410861969, + "selected_utility_gain_over_base": -0.03677475452423096, + "selector_regret": 0.06017845869064331, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08560334369211194, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5240784904076439, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2293184225071942, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.6871219206697284, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.7894833366296115, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007475614547729492, + "selected_utility_gain_over_base": 0.005546112894080579, + "selector_regret": 0.00242034625262022, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6907437905680791, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13291174173355103, + "selected_utility_gain_over_base": -1.867088258266449, + "selector_regret": 1.867088258266449, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.05319974288197324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.27232178375553634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.5644054573601021, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04821258783340454, + "selected_utility_gain_over_base": 0.012326281517744064, + "selector_regret": 0.027136601507663727, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2978540613458687, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.32358585873672097, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03574744984507561, + "selected_utility_gain_over_base": 0.02748026605695486, + "selector_regret": 0.02601233869791031, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.12037281951907768, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3177387291157021, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.31716416784041074, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01966226100921631, + "selected_utility_gain_over_base": -0.08317198604345322, + "selector_regret": 0.017348866909742355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1430027158285357, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.18624964231819074, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.4352129222811009, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00949472188949585, + "selected_utility_gain_over_base": -0.04462822154164314, + "selector_regret": 0.033287208527326584, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11061386076837161, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37949252862164845, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.7482380304201411, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08417382091283798, + "selected_utility_gain_over_base": -0.008823670446872711, + "selector_regret": 0.0273360013961792, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8162688843034812, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.10738897055995479, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.482628447985936, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08652313798666, + "selected_utility_gain_over_base": 0.03693946450948715, + "selector_regret": 1.91347686201334, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.48055562020716125, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.049984085020529354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15297338034841182, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18419977674091784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36738128595033803, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.719052615383995, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4863910843674545, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08180709183216095, + "selected_utility_gain_over_base": -1.918192908167839, + "selector_regret": 1.918192908167839, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5908952862576345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5935337022083772, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14234590579614123, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24600950331932453, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.25784951374924575, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23575927877927283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7082563723226508, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3721286611771941, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14139282703399658, + "selected_utility_gain_over_base": -0.04951559007167816, + "selector_regret": 0.07156434655189514, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20195660070480034, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10585396767858887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4886226110342108, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8246148228645325, + "selected_utility_gain_over_base": -1.1753851771354675, + "selector_regret": 1.1753851771354675, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23724048900553812, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3312653260052494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125, + "selected_utility_gain_over_base": -1.1875, + "selector_regret": 1.1875, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1978378634332485, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2917507478363105, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22513391077518463, + "selected_utility_gain_over_base": -0.3749566525220871, + "selector_regret": 0.005079537630081177, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33062341715384946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15758444275633177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.33329724085783513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6948683261871338, + "selected_utility_gain_over_base": -0.005086839199066162, + "selector_regret": 0.024266541004180908, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07057760540337482, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.502516346529802, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2277000706807752, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.31492302371466396, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06987287849187851, + "selected_utility_gain_over_base": -0.024030186235904694, + "selector_regret": 0.03010360151529312, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.715792002575413, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02248450182378292, + "selected_utility_gain_over_base": 0.020555000170134008, + "selector_regret": 0.016367433592677116, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6197611750356138, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06531327962875366, + "selected_utility_gain_over_base": -1.9346867203712463, + "selector_regret": 1.9346867203712463, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.047409745226937555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.21516109974286557, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.49594969906845865, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0682595744729042, + "selected_utility_gain_over_base": 0.03237326815724373, + "selector_regret": 0.2036382481455803, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2978540613458687, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.23981611288818955, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.09377718313008977, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2046539172174809, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3160658333588705, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.058951716870069504, + "selected_utility_gain_over_base": -0.04388253018260002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11621185424755208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2257704957099745, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.48035556946714403, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04980705305933952, + "selected_utility_gain_over_base": -0.004315890371799469, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1488380425829457, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3466799479100969, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.7566660574581419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.051346004009246826, + "selected_utility_gain_over_base": -0.04165148735046387, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.855453783986543, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.21986396235622255, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.45225824085777666, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.704480055086217, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03767126798629761, + "selected_utility_gain_over_base": -0.06684356927871704, + "selector_regret": 0.07543057203292847, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.049984085020529354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10853443168482588, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16442723445279572, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.445409885616724, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1215444803237915, + "selected_utility_gain_over_base": 0.015073955059051514, + "selector_regret": 1.8784555196762085, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.719052615383995, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13743820786476135, + "selected_utility_gain_over_base": 0.012670740485191345, + "selector_regret": 1.8625617921352386, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.47661080026664676, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6194446763134166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 5.424022674560547e-06, + "selected_utility_gain_over_base": 5.200505256652832e-06, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3427743477646694, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0997026115655899, + "selected_utility_gain_over_base": 0.012668535113334656, + "selector_regret": 1.90029738843441, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13062938104776378, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.181775355536573, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2414028495277438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22754916987219517, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6819725947753494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3899264429033258, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13791468739509583, + "selected_utility_gain_over_base": -0.05299372971057892, + "selector_regret": 0.0959680825471878, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20911211212626157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16360521054531033, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4915250634255605, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.21929223189854724, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3176403762902519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1785425841808319, + "selected_utility_gain_over_base": -1.821457415819168, + "selector_regret": 1.821457415819168, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.29031104002937436, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.6129032258064516, + "fallback_rate": 0.3870967741935484, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.4064516129032258, + "selected_success_gain_over_base": 0.12903225806451613, + "selected_utility": 0.8838888318235336, + "selector_regret": 0.34334768112874026, + "success_selector_gap": 0.16774193548387098, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "source_index": "data/cil_charts_rgb_refs/train/index.json", + "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6125, + "low": 0.4354166666666667, + "mean": 0.525, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.525, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6773333333333333, + "low": 0.18133333333333335, + "mean": 0.4413333333333334, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4413333333333334, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5733333333333334, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.6111111111111112, + "low": 0.4513888888888889, + "mean": 0.5277777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6125, + "low": 0.4354166666666667, + "mean": 0.525, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.525, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6773333333333333, + "low": 0.18133333333333335, + "mean": 0.4413333333333334, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4413333333333334, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5733333333333334, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.6111111111111112, + "low": 0.4513888888888889, + "mean": 0.5277777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5645833333333333, + "low": 0.3875, + "mean": 0.4749999999999999, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4749999999999999, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.0, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.75, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8186666666666668, + "low": 0.3226666666666667, + "mean": 0.5586666666666666, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5586666666666666, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.4266666666666667, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.5486111111111112, + "low": 0.3888888888888889, + "mean": 0.4722222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4430555555555556, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7206060606060607, + "low": 0.2206060606060606, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.368999757549798, + "low": 0.30105718122302094, + "mean": 0.33442069918328526, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33442069918328526, + "per_seed": { + "1073161778/train0": 0.34177985507477204, + "1073161778/train1": 0.4119603552284239, + "1073161778/train2": 0.33062341715384946, + "1113371261/train0": 0.2414028495277438, + "1113371261/train1": 0.25784951374924575, + "1113371261/train2": 0.2414028495277438, + "1151410346/train0": 0.19860467482965133, + "1151410346/train1": 0.20195660070480034, + "1151410346/train2": 0.20911211212626157, + "1261930456/train0": 0.38402002330650076, + "1261930456/train1": 0.4863910843674545, + "1261930456/train2": 0.47661080026664676, + "1324133122/train0": 0.15883484058846561, + "1324133122/train1": 0.10585396767858887, + "1324133122/train2": 0.16360521054531033, + "1324642411/train0": 0.4193019968970795, + "1324642411/train1": 0.36738128595033803, + "1324642411/train2": 0.445409885616724, + "1375492786/train0": 0.11744128190863853, + "1375492786/train1": 0.11061386076837161, + "1375492786/train2": 0.1488380425829457, + "141984615/train0": 0.3220294334917424, + "141984615/train1": 0.31716416784041074, + "141984615/train2": 0.3160658333588705, + "1461069713/train0": 0.3547798191536652, + "1461069713/train1": 0.6871219206697284, + "1461069713/train2": 0.31492302371466396, + "1480856810/train0": 0.3427743477646694, + "1480856810/train1": 0.5935337022083772, + "1480856810/train2": 0.3427743477646694, + "1509980664/train0": 0.6195026651718122, + "1509980664/train1": 0.7082563723226508, + "1509980664/train2": 0.6819725947753494, + "1522272686/train0": 0.0722437728966473, + "1522272686/train1": 0.08560334369211194, + "1522272686/train2": 0.07057760540337482, + "1549324805/train0": 0.7033632660299286, + "1549324805/train1": 0.48055562020716125, + "1549324805/train2": 0.704480055086217, + "1578990937/train0": 0.2150808050278072, + "1578990937/train1": 0.23575927877927283, + "1578990937/train2": 0.22754916987219517, + "1747178030/train0": 0.5849835367136861, + "1747178030/train1": 0.5240784904076439, + "1747178030/train2": 0.502516346529802, + "1790188787/train0": 0.7041875516262702, + "1790188787/train1": 0.719052615383995, + "1790188787/train2": 0.719052615383995, + "1822210471/train0": 0.049984085020529354, + "1822210471/train1": 0.049984085020529354, + "1822210471/train2": 0.049984085020529354, + "1845489013/train0": 0.24672540504885201, + "1845489013/train1": 0.2978540613458687, + "1845489013/train2": 0.2978540613458687, + "1999022574/train0": 0.2755208385556546, + "1999022574/train1": 0.3312653260052494, + "1999022574/train2": 0.3176403762902519, + "2012211287/train0": 0.17828726626426614, + "2012211287/train1": 0.20414937358844587, + "2012211287/train2": 0.1978378634332485, + "2038594191/train0": 0.3359117235345434, + "2038594191/train1": 0.37949252862164845, + "2038594191/train2": 0.3466799479100969, + "2054899126/train0": 0.8015567150083238, + "2054899126/train1": 0.7535063374357802, + "2054899126/train2": 0.7376074795110784, + "2061635888/train0": 0.13472633878966278, + "2061635888/train1": 0.14234590579614123, + "2061635888/train2": 0.13062938104776378, + "211245842/train0": 0.3392426447136978, + "211245842/train1": 0.3721286611771941, + "211245842/train2": 0.3899264429033258, + "278008524/train0": 0.48982706754483685, + "278008524/train1": 0.523516952673019, + "278008524/train2": 0.47410396996311766, + "307068879/train0": 0.28418063141808364, + "307068879/train1": 0.2759242006849786, + "307068879/train2": 0.2917507478363105, + "412102563/train0": 0.1966920683523119, + "412102563/train1": 0.23724048900553812, + "412102563/train2": 0.21929223189854724, + "440577711/train0": 0.18739562764215434, + "440577711/train1": 0.18419977674091784, + "440577711/train2": 0.16442723445279572, + "441046872/train0": 0.33329724085783513, + "441046872/train1": 0.3957300144448556, + "441046872/train2": 0.33329724085783513, + "509430392/train0": 0.5951794014623365, + "509430392/train1": 0.5908952862576345, + "509430392/train2": 0.6194446763134166, + "585904130/train0": 0.3083190727440344, + "585904130/train1": 0.3177387291157021, + "585904130/train2": 0.2046539172174809, + "608016290/train0": 0.1565404769152266, + "608016290/train1": 0.1646347051814787, + "608016290/train2": 0.15758444275633177, + "626486813/train0": 0.5564234962339261, + "626486813/train1": 0.5565345976831662, + "626486813/train2": 0.47780405773180123, + "636680137/train0": 0.40225098671598103, + "636680137/train1": 0.18624964231819074, + "636680137/train2": 0.2257704957099745, + "655884605/train0": 0.26463671358858426, + "655884605/train1": 0.24600950331932453, + "655884605/train2": 0.181775355536573, + "672328262/train0": 0.04820260893743538, + "672328262/train1": 0.05319974288197324, + "672328262/train2": 0.047409745226937555, + "701667486/train0": 0.5039098400899894, + "701667486/train1": 0.4886226110342108, + "701667486/train2": 0.4915250634255605, + "747260437/train0": 0.1675764505623386, + "747260437/train1": 0.10738897055995479, + "747260437/train2": 0.21986396235622255, + "750448296/train0": 0.37880623603795066, + "750448296/train1": 0.34807137553236295, + "750448296/train2": 0.32713744779683995, + "767747424/train0": 0.2970760855424009, + "767747424/train1": 0.2643064870157273, + "767747424/train2": 0.25424848075521866 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4060311257217533, + "low": 0.173648328139981, + "mean": 0.2948498749390142, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2948498749390142, + "per_task": { + "LiftPegUpright-v1": 0.27050518877972946, + "PickCube-v1": 0.07614157399737802, + "PullCube-v1": 0.3693117299280415, + "PushCube-v1": 0.4634462366336652, + "StackCube-v1": 0.29484464535625693 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.3823460563231854, + "low": 0.3152776212965019, + "mean": 0.3476131101003723, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6722222222222223, + "low": 0.5055555555555555, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7430303030303029, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4930555555555556, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4116970675635254, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5246927151765788, + "low": 0.5848668839583795, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.360821089959548, + "low": 1.0729441113071516, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 6.332638888888889, + "low": 4.827083333333333, + "mean": 5.577083333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 5.577083333333334, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 7.0, + "1073161778/train2": 6.0, + "1113371261/train0": 0.0, + "1113371261/train1": 4.0, + "1113371261/train2": 2.0, + "1151410346/train0": 13.0, + "1151410346/train1": 9.0, + "1151410346/train2": 11.0, + "1261930456/train0": 5.0, + "1261930456/train1": 11.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 0.0, + "1324642411/train0": 12.0, + "1324642411/train1": 9.0, + "1324642411/train2": 14.0, + "1375492786/train0": 12.0, + "1375492786/train1": 10.0, + "1375492786/train2": 0.0, + "141984615/train0": 4.0, + "141984615/train1": 1.0, + "141984615/train2": 11.0, + "1461069713/train0": 2.0, + "1461069713/train1": 0.0, + "1461069713/train2": 5.0, + "1480856810/train0": 1.0, + "1480856810/train1": 13.0, + "1480856810/train2": 5.0, + "1509980664/train0": 9.0, + "1509980664/train1": 4.0, + "1509980664/train2": 12.0, + "1522272686/train0": 7.0, + "1522272686/train1": 7.0, + "1522272686/train2": 8.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 2.0, + "1578990937/train0": 6.0, + "1578990937/train1": 12.0, + "1578990937/train2": 0.0, + "1747178030/train0": 4.0, + "1747178030/train1": 1.0, + "1747178030/train2": 0.0, + "1790188787/train0": 5.0, + "1790188787/train1": 6.0, + "1790188787/train2": 8.0, + "1822210471/train0": 4.0, + "1822210471/train1": 4.0, + "1822210471/train2": 10.0, + "1845489013/train0": 10.0, + "1845489013/train1": 3.0, + "1845489013/train2": 11.0, + "1999022574/train0": 5.0, + "1999022574/train1": 9.0, + "1999022574/train2": 6.0, + "2012211287/train0": 10.0, + "2012211287/train1": 15.0, + "2012211287/train2": 4.0, + "2038594191/train0": 0.0, + "2038594191/train1": 4.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 3.5, + "2061635888/train0": 5.0, + "2061635888/train1": 8.0, + "2061635888/train2": 5.0, + "211245842/train0": 11.0, + "211245842/train1": 15.0, + "211245842/train2": 15.0, + "278008524/train0": 5.5, + "278008524/train1": 2.5, + "278008524/train2": 9.5, + "307068879/train0": 2.0, + "307068879/train1": 7.0, + "307068879/train2": 0.0, + "412102563/train0": 13.0, + "412102563/train1": 10.0, + "412102563/train2": 4.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 7.0, + "441046872/train0": 10.0, + "441046872/train1": 3.0, + "441046872/train2": 6.0, + "509430392/train0": 14.0, + "509430392/train1": 3.0, + "509430392/train2": 10.0, + "585904130/train0": 1.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 15.0, + "608016290/train1": 9.0, + "608016290/train2": 9.0, + "626486813/train0": 1.5, + "626486813/train1": 0.5, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 2.0, + "636680137/train2": 0.0, + "655884605/train0": 5.0, + "655884605/train1": 2.0, + "655884605/train2": 3.0, + "672328262/train0": 2.0, + "672328262/train1": 9.0, + "672328262/train2": 7.0, + "701667486/train0": 7.0, + "701667486/train1": 7.0, + "701667486/train2": 5.0, + "747260437/train0": 0.0, + "747260437/train1": 8.0, + "747260437/train2": 1.0, + "750448296/train0": 3.3333333333333335, + "750448296/train1": 8.333333333333334, + "750448296/train2": 5.333333333333333, + "767747424/train0": 9.5, + "767747424/train1": 8.0, + "767747424/train2": 5.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 7.25111111111111, + "low": 5.334545454545454, + "mean": 6.505656565656567, + "n": 5, + "num_samples": 1000 + }, + "mean": 6.505656565656567, + "per_task": { + "LiftPegUpright-v1": 7.266666666666667, + "PickCube-v1": 7.333333333333333, + "PullCube-v1": 4.266666666666667, + "PushCube-v1": 7.055555555555555, + "StackCube-v1": 6.606060606060606 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 6.256944444444445, + "low": 4.798611111111111, + "mean": 5.527777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.36041666666666666, + "low": 0.20208333333333334, + "mean": 0.28125, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.28125, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2773333333333333, + "low": 0.057777777777777775, + "mean": 0.16953535353535354, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.16953535353535354, + "per_task": { + "LiftPegUpright-v1": 0.06666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.2222222222222222, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3333333333333333, + "low": 0.19444444444444445, + "mean": 0.2638888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.025, + "low": -0.07916666666666666, + "mean": -0.025, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.025, + "per_seed": { + "1073161778/train0": -1.0, + "1073161778/train1": -1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": -1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": -1.0, + "1999022574/train2": -1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": -1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.019555555555555555, + "low": -0.08969696969696969, + "mean": -0.030343434343434346, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.030343434343434346, + "per_task": { + "LiftPegUpright-v1": -0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.013333333333333334, + "PushCube-v1": 0.05555555555555555, + "StackCube-v1": -0.06060606060606061 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.020833333333333332, + "low": -0.08333333333333333, + "mean": -0.027777777777777776, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8936206722554442, + "low": 0.6056815767916203, + "mean": 0.7508006510787204, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7508006510787204, + "per_seed": { + "1073161778/train0": 0.67014479637146, + "1073161778/train1": 0.6916947960853577, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 0.08180709183216095, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 0.1280023753643036, + "1324642411/train2": 0.1215444803237915, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.006893515586853027, + "141984615/train1": 0.01966226100921631, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.009687979705631733, + "1461069713/train2": 0.06987287849187851, + "1480856810/train0": 0.06508849561214447, + "1480856810/train1": 0.20681259036064148, + "1480856810/train2": 0.0997026115655899, + "1509980664/train0": 0.8125001788139343, + "1509980664/train1": 0.21804393827915192, + "1509980664/train2": 0.8189421892166138, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.0303483996540308, + "1549324805/train2": 0.03767126798629761, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 0.13741976022720337, + "1790188787/train2": 0.13743820786476135, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 0.8125, + "1999022574/train2": 0.1785425841808319, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.052448581904173, + "2054899126/train1": 1.0664558708667755, + "2054899126/train2": 1.0326566398143768, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 2.0, + "211245842/train1": 0.14139282703399658, + "211245842/train2": 0.13791468739509583, + "278008524/train0": 0.0616548378020525, + "278008524/train1": 0.06736786291003227, + "278008524/train2": 1.034129787236452, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.22513391077518463, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6693241000175476, + "441046872/train1": 0.663180410861969, + "441046872/train2": 0.6948683261871338, + "509430392/train0": 0.12097799777984619, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 0.05923032760620117, + "585904130/train1": 2.0, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.005386819248087704, + "626486813/train1": 0.02161153219640255, + "626486813/train2": 0.015375842805951834, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 0.8246148228645325, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.1003921168545882, + "750448296/train1": 0.11465192586183548, + "750448296/train2": 0.1037093202273051, + "767747424/train0": 0.5294794533401728, + "767747424/train1": 0.562042286619544, + "767747424/train2": 0.572120369412005 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7290419181390015, + "low": 0.5773350460330645, + "mean": 0.6533754614969582, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6533754614969582, + "per_task": { + "LiftPegUpright-v1": 0.7441632241010666, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7376457962176453, + "PushCube-v1": 0.5195176344778802, + "StackCube-v1": 0.6815915500575845 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8271019692199642, + "low": 0.5659115896123694, + "mean": 0.6950111038766207, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.04744749259488243, + "low": -0.1336556534100156, + "mean": -0.039317411219073825, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.039317411219073825, + "per_seed": { + "1073161778/train0": -1.32985520362854, + "1073161778/train1": -1.3083052039146423, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": -1.918192908167839, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.03758952021598816, + "1324642411/train1": 0.0215318500995636, + "1324642411/train2": 0.015073955059051514, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.0959407314658165, + "141984615/train1": -0.08317198604345322, + "141984615/train2": -0.04388253018260002, + "1461069713/train0": -0.03370469808578491, + "1461069713/train1": -0.08421508502215147, + "1461069713/train2": -0.024030186235904694, + "1480856810/train0": -0.02194558084011078, + "1480856810/train1": 0.11977851390838623, + "1480856810/train2": 0.012668535113334656, + "1509980664/train0": -0.02090984582901001, + "1509980664/train1": -0.6153660863637924, + "1509980664/train2": -0.014467835426330566, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": -0.07416643761098385, + "1549324805/train2": -0.06684356927871704, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.87523253262043, + "1790188787/train1": 0.012652292847633362, + "1790188787/train2": 0.012670740485191345, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": -1.1875, + "1999022574/train2": -1.821457415819168, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9475514180958271, + "2054899126/train1": -0.9335441291332245, + "2054899126/train2": -0.9673433601856232, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.8090915828943253, + "211245842/train1": -0.04951559007167816, + "211245842/train2": -0.05299372971057892, + "278008524/train0": 0.018919847905635834, + "278008524/train1": 0.024632873013615608, + "278008524/train2": 0.9913947973400354, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": -0.3749566525220871, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": -0.030631065368652344, + "441046872/train1": -0.03677475452423096, + "441046872/train2": -0.005086839199066162, + "509430392/train0": 0.12097777426242828, + "509430392/train1": 1.999999776482582, + "509430392/train2": 5.200505256652832e-06, + "585904130/train0": -0.1224852204322815, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0002884765272028744, + "626486813/train1": 0.01651318947551772, + "626486813/train2": 0.010277500085067004, + "636680137/train0": -0.027042903006076813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": -1.1753851771354675, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": -0.017201032489538193, + "750448296/train1": -0.0029412234822909036, + "750448296/train2": -0.013883829116821289, + "767747424/train0": -0.04371988866478205, + "767747424/train1": -0.011157055385410786, + "767747424/train2": -0.0010789725929498672 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0498025095248595, + "low": -0.13914316054847503, + "mean": -0.04218433567705667, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.04218433567705667, + "per_task": { + "LiftPegUpright-v1": -0.20570731461048125, + "PickCube-v1": 0.0, + "PullCube-v1": -0.03752466500581553, + "PushCube-v1": 0.1271134556995498, + "StackCube-v1": -0.09480315446853638 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.034699834853300776, + "low": -0.13894624607766876, + "mean": -0.04680848256571658, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7080093832621869, + "low": 0.4333724000698162, + "mean": 0.5641869812521488, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5641869812521488, + "per_seed": { + "1073161778/train0": 1.32985520362854, + "1073161778/train1": 1.3083052039146423, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 1.918192908167839, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.0, + "1324642411/train1": 1.8719976246356964, + "1324642411/train2": 1.8784555196762085, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.008284389972686768, + "141984615/train1": 0.017348866909742355, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.00689703319221735, + "1461069713/train2": 0.03010360151529312, + "1480856810/train0": 1.9349115043878555, + "1480856810/train1": 1.7931874096393585, + "1480856810/train2": 1.90029738843441, + "1509980664/train0": 1.1874998211860657, + "1509980664/train1": 1.781956061720848, + "1509980664/train2": 1.1810578107833862, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.16857311688363552, + "1549324805/train2": 0.07543057203292847, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.8625802397727966, + "1790188787/train2": 1.8625617921352386, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.1875, + "1999022574/train2": 1.821457415819168, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.024834569543600082, + "2054899126/train1": 0.9335441291332245, + "2054899126/train2": 0.9673433601856232, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 0.0, + "211245842/train1": 0.07156434655189514, + "211245842/train2": 0.0959680825471878, + "278008524/train0": 1.0117014851421118, + "278008524/train1": 0.9703067317605019, + "278008524/train2": 0.10181912407279015, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.005079537630081177, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 1.3306758999824524, + "441046872/train1": 0.06017845869064331, + "441046872/train2": 0.024266541004180908, + "509430392/train0": 1.8790220022201538, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.9407696723937988, + "585904130/train1": 0.0, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.03930865635629743, + "626486813/train1": 0.014216342475265265, + "626486813/train2": 0.029523412231355906, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.6498691290616989, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.1753851771354675, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6522682110468546, + "750448296/train1": 1.2558513482411702, + "750448296/train2": 1.246739347775777, + "767747424/train0": 0.5129497344605625, + "767747424/train1": 0.5009108046069741, + "767747424/train2": 0.49504672549664974 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9451237162535051, + "low": 0.22875468119916817, + "mean": 0.5777667584602693, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5777667584602693, + "per_task": { + "LiftPegUpright-v1": 0.6957427720228831, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4480306339729577, + "PushCube-v1": 1.266263780494531, + "StackCube-v1": 0.4787966058109746 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.7071303886720367, + "low": 0.4517241518980073, + "mean": 0.5738296886734638, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.41388888888888886, + "low": 0.24930555555555556, + "mean": 0.3284722222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3284722222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5333333333333333, + "low": 0.13599999999999998, + "mean": 0.34593939393939394, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.34593939393939394, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.22666666666666666, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.4097222222222222, + "low": 0.2569444444444444, + "mean": 0.3263888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30416666666666664, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7191919191919192, + "low": 0.13511111111111113, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2638888888888889, + "low": 0.1388888888888889, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5434736852068454, + "low": 0.2965679521788843, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.330930626693398, + "low": 0.24626440485649642, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.485598717091812, + "low": 0.272009730946997, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2903110400293735, + "low": 0.2903110400293735, + "mean": 0.2903110400293735, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.2903110400293735, + "per_seed": { + "1073161778/train0": 0.29031104002937436, + "1073161778/train1": 0.29031104002937436, + "1073161778/train2": 0.29031104002937436, + "1113371261/train0": 0.29031104002937436, + "1113371261/train1": 0.29031104002937436, + "1113371261/train2": 0.29031104002937436, + "1151410346/train0": 0.29031104002937436, + "1151410346/train1": 0.29031104002937436, + "1151410346/train2": 0.29031104002937436, + "1261930456/train0": 0.29031104002937436, + "1261930456/train1": 0.29031104002937436, + "1261930456/train2": 0.29031104002937436, + "1324133122/train0": 0.29031104002937436, + "1324133122/train1": 0.29031104002937436, + "1324133122/train2": 0.29031104002937436, + "1324642411/train0": 0.29031104002937436, + "1324642411/train1": 0.29031104002937436, + "1324642411/train2": 0.29031104002937436, + "1375492786/train0": 0.29031104002937436, + "1375492786/train1": 0.29031104002937436, + "1375492786/train2": 0.29031104002937436, + "141984615/train0": 0.29031104002937436, + "141984615/train1": 0.29031104002937436, + "141984615/train2": 0.29031104002937436, + "1461069713/train0": 0.29031104002937436, + "1461069713/train1": 0.29031104002937436, + "1461069713/train2": 0.29031104002937436, + "1480856810/train0": 0.29031104002937436, + "1480856810/train1": 0.29031104002937436, + "1480856810/train2": 0.29031104002937436, + "1509980664/train0": 0.29031104002937436, + "1509980664/train1": 0.29031104002937436, + "1509980664/train2": 0.29031104002937436, + "1522272686/train0": 0.29031104002937436, + "1522272686/train1": 0.29031104002937436, + "1522272686/train2": 0.29031104002937436, + "1549324805/train0": 0.29031104002937436, + "1549324805/train1": 0.29031104002937436, + "1549324805/train2": 0.29031104002937436, + "1578990937/train0": 0.29031104002937436, + "1578990937/train1": 0.29031104002937436, + "1578990937/train2": 0.29031104002937436, + "1747178030/train0": 0.29031104002937436, + "1747178030/train1": 0.29031104002937436, + "1747178030/train2": 0.29031104002937436, + "1790188787/train0": 0.29031104002937436, + "1790188787/train1": 0.29031104002937436, + "1790188787/train2": 0.29031104002937436, + "1822210471/train0": 0.29031104002937436, + "1822210471/train1": 0.29031104002937436, + "1822210471/train2": 0.29031104002937436, + "1845489013/train0": 0.29031104002937436, + "1845489013/train1": 0.29031104002937436, + "1845489013/train2": 0.29031104002937436, + "1999022574/train0": 0.29031104002937436, + "1999022574/train1": 0.29031104002937436, + "1999022574/train2": 0.29031104002937436, + "2012211287/train0": 0.29031104002937436, + "2012211287/train1": 0.29031104002937436, + "2012211287/train2": 0.29031104002937436, + "2038594191/train0": 0.29031104002937436, + "2038594191/train1": 0.29031104002937436, + "2038594191/train2": 0.29031104002937436, + "2054899126/train0": 0.29031104002937436, + "2054899126/train1": 0.29031104002937436, + "2054899126/train2": 0.29031104002937436, + "2061635888/train0": 0.29031104002937436, + "2061635888/train1": 0.29031104002937436, + "2061635888/train2": 0.29031104002937436, + "211245842/train0": 0.29031104002937436, + "211245842/train1": 0.29031104002937436, + "211245842/train2": 0.29031104002937436, + "278008524/train0": 0.29031104002937436, + "278008524/train1": 0.29031104002937436, + "278008524/train2": 0.29031104002937436, + "307068879/train0": 0.29031104002937436, + "307068879/train1": 0.29031104002937436, + "307068879/train2": 0.29031104002937436, + "412102563/train0": 0.29031104002937436, + "412102563/train1": 0.29031104002937436, + "412102563/train2": 0.29031104002937436, + "440577711/train0": 0.29031104002937436, + "440577711/train1": 0.29031104002937436, + "440577711/train2": 0.29031104002937436, + "441046872/train0": 0.29031104002937436, + "441046872/train1": 0.29031104002937436, + "441046872/train2": 0.29031104002937436, + "509430392/train0": 0.29031104002937436, + "509430392/train1": 0.29031104002937436, + "509430392/train2": 0.29031104002937436, + "585904130/train0": 0.29031104002937436, + "585904130/train1": 0.29031104002937436, + "585904130/train2": 0.29031104002937436, + "608016290/train0": 0.29031104002937436, + "608016290/train1": 0.29031104002937436, + "608016290/train2": 0.29031104002937436, + "626486813/train0": 0.29031104002937436, + "626486813/train1": 0.29031104002937436, + "626486813/train2": 0.29031104002937436, + "636680137/train0": 0.29031104002937436, + "636680137/train1": 0.29031104002937436, + "636680137/train2": 0.29031104002937436, + "655884605/train0": 0.29031104002937436, + "655884605/train1": 0.29031104002937436, + "655884605/train2": 0.29031104002937436, + "672328262/train0": 0.29031104002937436, + "672328262/train1": 0.29031104002937436, + "672328262/train2": 0.29031104002937436, + "701667486/train0": 0.29031104002937436, + "701667486/train1": 0.29031104002937436, + "701667486/train2": 0.29031104002937436, + "747260437/train0": 0.29031104002937436, + "747260437/train1": 0.29031104002937436, + "747260437/train2": 0.29031104002937436, + "750448296/train0": 0.29031104002937436, + "750448296/train1": 0.29031104002937436, + "750448296/train2": 0.29031104002937436, + "767747424/train0": 0.29031104002937436, + "767747424/train1": 0.29031104002937436, + "767747424/train2": 0.29031104002937436 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2903110400293744, + "low": 0.29031104002937413, + "mean": 0.29031104002937436, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.29031104002937436, + "per_task": { + "LiftPegUpright-v1": 0.2903110400293744, + "PickCube-v1": 0.29031104002937436, + "PullCube-v1": 0.29031104002937397, + "PushCube-v1": 0.2903110400293744, + "StackCube-v1": 0.2903110400293744 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.29031104002937336, + "low": 0.29031104002937336, + "mean": 0.29031104002937336, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": 0.29031104002937436 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..c0f30a5f71d1aea6b3143c96b5c4a459ae93e095 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.34177985507477204, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.67014479637146, + "selected_utility_gain_over_base": -1.32985520362854, + "selector_regret": 1.32985520362854, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4119603552284239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6916947960853577, + "selected_utility_gain_over_base": -1.3083052039146423, + "selector_regret": 1.3083052039146423, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33062341715384946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2414028495277438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.29031104002937436 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.25784951374924575, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.29031104002937436 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2414028495277438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.29031104002937436 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19860467482965133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.29031104002937436 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20195660070480034, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20911211212626157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.29031104002937436 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38402002330650076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4863910843674545, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08180709183216095, + "selected_utility_gain_over_base": -1.918192908167839, + "selector_regret": 1.918192908167839, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.47661080026664676, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15883484058846561, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.29031104002937436 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10585396767858887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.29031104002937436 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16360521054531033, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.29031104002937436 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4193019968970795, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14406004548072815, + "selected_utility_gain_over_base": 0.03758952021598816, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.29031104002937436 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36738128595033803, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.445409885616724, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1215444803237915, + "selected_utility_gain_over_base": 0.015073955059051514, + "selector_regret": 1.8784555196762085, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11744128190863853, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11061386076837161, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1488380425829457, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3220294334917424, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006893515586853027, + "selected_utility_gain_over_base": -0.0959407314658165, + "selector_regret": 0.008284389972686768, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.29031104002937436 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.31716416784041074, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01966226100921631, + "selected_utility_gain_over_base": -0.08317198604345322, + "selector_regret": 0.017348866909742355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.29031104002937436 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3160658333588705, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.058951716870069504, + "selected_utility_gain_over_base": -0.04388253018260002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.29031104002937436 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3547798191536652, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06019836664199829, + "selected_utility_gain_over_base": -0.03370469808578491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.29031104002937436 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.6871219206697284, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.29031104002937436 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.31492302371466396, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06987287849187851, + "selected_utility_gain_over_base": -0.024030186235904694, + "selector_regret": 0.03010360151529312, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.29031104002937436 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3427743477646694, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06508849561214447, + "selected_utility_gain_over_base": -0.02194558084011078, + "selector_regret": 1.9349115043878555, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5935337022083772, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3427743477646694, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0997026115655899, + "selected_utility_gain_over_base": 0.012668535113334656, + "selector_regret": 1.90029738843441, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6195026651718122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8125001788139343, + "selected_utility_gain_over_base": -0.02090984582901001, + "selector_regret": 1.1874998211860657, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7082563723226508, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6819725947753494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0722437728966473, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.29031104002937436 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08560334369211194, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.29031104002937436 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07057760540337482, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.29031104002937436 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.7033632660299286, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.29031104002937436 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.48055562020716125, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0303483996540308, + "selected_utility_gain_over_base": -0.07416643761098385, + "selector_regret": 0.16857311688363552, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.29031104002937436 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.704480055086217, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03767126798629761, + "selected_utility_gain_over_base": -0.06684356927871704, + "selector_regret": 0.07543057203292847, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.29031104002937436 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2150808050278072, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23575927877927283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22754916987219517, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5849835367136861, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5240784904076439, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.502516346529802, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7041875516262702, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.719052615383995, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.719052615383995, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13743820786476135, + "selected_utility_gain_over_base": 0.012670740485191345, + "selector_regret": 1.8625617921352386, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.049984085020529354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.049984085020529354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.049984085020529354, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24672540504885201, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2978540613458687, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2978540613458687, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2755208385556546, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3312653260052494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125, + "selected_utility_gain_over_base": -1.1875, + "selector_regret": 1.1875, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3176403762902519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1785425841808319, + "selected_utility_gain_over_base": -1.821457415819168, + "selector_regret": 1.821457415819168, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17828726626426614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20414937358844587, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1978378634332485, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3359117235345434, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37949252862164845, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3466799479100969, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8015567150083238, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.052448581904173, + "selected_utility_gain_over_base": -0.9475514180958271, + "selector_regret": 0.024834569543600082, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.29031104002937436 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7535063374357802, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0664558708667755, + "selected_utility_gain_over_base": -0.9335441291332245, + "selector_regret": 0.9335441291332245, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7376074795110784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0326566398143768, + "selected_utility_gain_over_base": -0.9673433601856232, + "selector_regret": 0.9673433601856232, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13472633878966278, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.29031104002937436 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14234590579614123, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.29031104002937436 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13062938104776378, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.29031104002937436 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3392426447136978, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8090915828943253, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3721286611771941, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14139282703399658, + "selected_utility_gain_over_base": -0.04951559007167816, + "selector_regret": 0.07156434655189514, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.29031104002937436 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3899264429033258, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13791468739509583, + "selected_utility_gain_over_base": -0.05299372971057892, + "selector_regret": 0.0959680825471878, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.29031104002937436 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.48982706754483685, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 5.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0616548378020525, + "selected_utility_gain_over_base": 0.018919847905635834, + "selector_regret": 1.0117014851421118, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.29031104002937436 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.523516952673019, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 2.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06736786291003227, + "selected_utility_gain_over_base": 0.024632873013615608, + "selector_regret": 0.9703067317605019, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.29031104002937436 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.47410396996311766, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 9.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.034129787236452, + "selected_utility_gain_over_base": 0.9913947973400354, + "selector_regret": 0.10181912407279015, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28418063141808364, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.29031104002937436 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2759242006849786, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.29031104002937436 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2917507478363105, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22513391077518463, + "selected_utility_gain_over_base": -0.3749566525220871, + "selector_regret": 0.005079537630081177, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.29031104002937436 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1966920683523119, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23724048900553812, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.21929223189854724, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18739562764215434, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18419977674091784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16442723445279572, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.33329724085783513, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6693241000175476, + "selected_utility_gain_over_base": -0.030631065368652344, + "selector_regret": 1.3306758999824524, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3957300144448556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.663180410861969, + "selected_utility_gain_over_base": -0.03677475452423096, + "selector_regret": 0.06017845869064331, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.29031104002937436 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.33329724085783513, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6948683261871338, + "selected_utility_gain_over_base": -0.005086839199066162, + "selector_regret": 0.024266541004180908, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.29031104002937436 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5951794014623365, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5908952862576345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6194446763134166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 5.424022674560547e-06, + "selected_utility_gain_over_base": 5.200505256652832e-06, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.29031104002937436 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3083190727440344, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05923032760620117, + "selected_utility_gain_over_base": -0.1224852204322815, + "selector_regret": 1.9407696723937988, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3177387291157021, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2046539172174809, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1565404769152266, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1646347051814787, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15758444275633177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.29031104002937436 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.5564234962339261, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 1.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.005386819248087704, + "selected_utility_gain_over_base": 0.0002884765272028744, + "selector_regret": 0.03930865635629743, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.29031104002937436 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.5565345976831662, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02161153219640255, + "selected_utility_gain_over_base": 0.01651318947551772, + "selector_regret": 0.014216342475265265, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.29031104002937436 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.47780405773180123, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015375842805951834, + "selected_utility_gain_over_base": 0.010277500085067004, + "selector_regret": 0.029523412231355906, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.29031104002937436 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.40225098671598103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.29031104002937436 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.18624964231819074, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.29031104002937436 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2257704957099745, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.29031104002937436 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.26463671358858426, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.29031104002937436 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24600950331932453, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.29031104002937436 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.181775355536573, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.29031104002937436 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04820260893743538, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.05319974288197324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.047409745226937555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5039098400899894, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4886226110342108, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8246148228645325, + "selected_utility_gain_over_base": -1.1753851771354675, + "selector_regret": 1.1753851771354675, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4915250634255605, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.29031104002937436 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.1675764505623386, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.29031104002937436 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.10738897055995479, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.29031104002937436 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.21986396235622255, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.29031104002937436 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.37880623603795066, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 3.3333333333333335, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1003921168545882, + "selected_utility_gain_over_base": -0.017201032489538193, + "selector_regret": 0.6522682110468546, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.29031104002937436 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.34807137553236295, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 8.333333333333334, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11465192586183548, + "selected_utility_gain_over_base": -0.0029412234822909036, + "selector_regret": 1.2558513482411702, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.29031104002937436 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.32713744779683995, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 5.333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1037093202273051, + "selected_utility_gain_over_base": -0.013883829116821289, + "selector_regret": 1.246739347775777, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.29031104002937436 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.2970760855424009, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 9.5, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5294794533401728, + "selected_utility_gain_over_base": -0.04371988866478205, + "selector_regret": 0.5129497344605625, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.29031104002937436 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.2643064870157273, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 8.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.562042286619544, + "selected_utility_gain_over_base": -0.011157055385410786, + "selector_regret": 0.5009108046069741, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.29031104002937436 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.25424848075521866, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 5.25, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.572120369412005, + "selected_utility_gain_over_base": -0.0010789725929498672, + "selector_regret": 0.49504672549664974, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.29031104002937436 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..faf457bc53bbf39259af2358ccf3412e2ff002e6 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.4666666666666667, + "execute_generated": 0.4666666666666667, + "fallback_rate": 0.5333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "predicted_margin": 0.27050518877972946, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 7.266666666666667, + "selected_success": 0.06666666666666667, + "selected_success_gain_over_base": -0.13333333333333333, + "selected_utility": 0.7441632241010666, + "selected_utility_gain_over_base": -0.20570731461048125, + "selector_regret": 0.6957427720228831, + "success_selector_gap": 0.5333333333333333, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.2903110400293744 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07614157399737802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 7.333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.29031104002937436 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.5733333333333334, + "execute_generated": 0.5733333333333334, + "fallback_rate": 0.4266666666666667, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.3693117299280415, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 4.266666666666667, + "selected_success": 0.3466666666666667, + "selected_success_gain_over_base": -0.013333333333333334, + "selected_utility": 0.7376457962176453, + "selected_utility_gain_over_base": -0.03752466500581553, + "selector_regret": 0.4480306339729577, + "success_selector_gap": 0.22666666666666666, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.29031104002937397 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.8333333333333334, + "execute_generated": 0.8333333333333334, + "fallback_rate": 0.16666666666666666, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "predicted_margin": 0.4634462366336652, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 7.055555555555555, + "selected_success": 0.2222222222222222, + "selected_success_gain_over_base": 0.05555555555555555, + "selected_utility": 0.5195176344778802, + "selected_utility_gain_over_base": 0.1271134556995498, + "selector_regret": 1.266263780494531, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.2903110400293744 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "predicted_margin": 0.29484464535625693, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 6.606060606060606, + "selected_success": 0.21212121212121213, + "selected_success_gain_over_base": -0.06060606060606061, + "selected_utility": 0.6815915500575845, + "selected_utility_gain_over_base": -0.09480315446853638, + "selector_regret": 0.4787966058109746, + "success_selector_gap": 0.30303030303030304, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.2903110400293744 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..ac156b511a32dff7b4f7437f27918a5d216f5fcb --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.290311` +Feature set: `chart_source_compat` +Target: `utility_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4982 | 0.5018 | 0.2780 | 0.4657 | 0.5343 | 0.5343 | 0.1733 | 0.1191 | +| selection | 0.6129 | 0.3871 | 0.2774 | 0.4065 | 0.5419 | 0.5677 | 0.1806 | 0.1677 | +| eval | 0.5278 | 0.4722 | 0.2917 | 0.2639 | 0.5694 | 0.5486 | 0.2014 | 0.3264 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..2706c3e0547971eab692012771ffa1c44982d2e5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,20 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + }, + "source_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..69131ba7a848200b2722870750881387569f3f7e --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.5278 & 0.4722 & 0.2917 & 0.2639 & 0.5694 & 0.5486 & 0.2014 & 0.3264 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a599c13449149c93cad6d9c6b86010a5655763ee --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0} +tau=0.290311 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..45538ee559f5a3ddc46300665f6fc1cf1c0a5965 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test --feature-set chart_compat --selector-chart-feature-mode base_context_obs --target positive_margin --k 16 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e75b60696832a5e0e8d19f05d4bf1a33bf28d13 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,15 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_compat +k: 16 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test +seed: 0 +selection_frac: 0.35 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +target: positive_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c001552e076e38f762c1980a9ac2fdd590af6dc4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0d42ce25e01f930916b0aaca3c292b7823623a42 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,7470 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.3541666666666667, + "fallback_rate": 0.6458333333333334, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.2847222222222222, + "selected_success_gain_over_base": -0.006944444444444444, + "selected_utility": 0.7162999392102292, + "selector_regret": 0.5429793549620727, + "success_selector_gap": 0.2986111111111111, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "chart_compat", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4548736462093863, + "fallback_rate": 0.5451263537906137, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.5342960288808665, + "proposal_oracle_success": 0.5342960288808665, + "proposal_oracle_utility": 1.1336619752116286, + "selected_success": 0.4620938628158845, + "selected_success_gain_over_base": 0.18411552346570398, + "selected_utility": 1.0263910738352875, + "selector_regret": 0.2268675415909476, + "success_selector_gap": 0.1263537906137184, + "success_support_gap": 0.17328519855595667, + "support_gap": 0.3521794904517649 + }, + "k": 16, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.49032258064516127, + "fallback_rate": 0.5096774193548387, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.44516129032258067, + "selected_success_gain_over_base": 0.16774193548387098, + "selected_utility": 0.9639207383876877, + "selector_regret": 0.2880004003505042, + "success_selector_gap": 0.14193548387096774, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "num_fit_candidates": 4175, + "num_fit_rows": 277, + "num_selection_candidates": 2335, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.08991720318616597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2797080480367002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28355827608242967, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18560038716365618, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20172868695503152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08863633481483887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3352371313866231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4110900404587235, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.023549815639853477, + "selected_utility_gain_over_base": -0.13813611306250095, + "selector_regret": 1.9764501843601465, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3050981300072402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.6362267958435123, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0025064547080546618, + "selected_utility_gain_over_base": 0.0005769530544057488, + "selector_regret": 0.007676919689401984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.507461874569938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1048971638083458, + "selected_utility_gain_over_base": -1.8951028361916542, + "selector_regret": 0.049669139087200165, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042802515015988585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.17492469965849014, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.35986959125133827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09716508537530899, + "selected_utility_gain_over_base": 0.061278779059648514, + "selector_regret": 0.04954756051301956, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22822097403519162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.19016637277131918, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.26403852161187397, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26430287849596273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3988789611600871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0042244791984558105, + "selected_utility_gain_over_base": -0.09860976785421371, + "selector_regret": 0.010953426361083984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34069303977256765, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371194860392053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.2726537010014917, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16043123876023302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05387979745864868, + "selected_utility_gain_over_base": -1.9461202025413513, + "selector_regret": 1.9461202025413513, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.615562919823872, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019754549488425255, + "selected_utility_gain_over_base": -0.07324294187128544, + "selector_regret": 0.02163984440267086, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46618538705348106, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.28297507531186483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.471092511551753, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.4534903350362056, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10685249697353241, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3010393386519421, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244761772076728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.061532557010650635, + "selected_utility_gain_over_base": -0.044937968254089355, + "selector_regret": 0.08252748847007751, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7256808008161852, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14965972304344177, + "selected_utility_gain_over_base": 0.024892255663871765, + "selector_regret": 1.8503402769565582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5566373654374286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.39582323274190295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16138880459399857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30368062376907057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32662525863909614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2255580985362039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5870819789979614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36446911237871227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11833827197551727, + "selected_utility_gain_over_base": -0.07257014513015747, + "selector_regret": 1.8816617280244827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20532758727946646, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1953526650841092, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3589247640152091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20585303592937382, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2830091920285253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1018745030033766, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24778606153341362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3337636013451598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2379598925580104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24443771108596596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09154681464635614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33370414261502274, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3705938518118879, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10141054540872574, + "selected_utility_gain_over_base": -0.06027538329362869, + "selector_regret": 1.8985894545912743, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.7484074094930177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.6387349542040327, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007475614547729492, + "selected_utility_gain_over_base": 0.005546112894080579, + "selector_regret": 0.00242034625262022, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41414798691644433, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13291174173355103, + "selected_utility_gain_over_base": -1.867088258266449, + "selector_regret": 1.867088258266449, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04466194101303914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.17110779360440906, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.4007460177868561, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04821258783340454, + "selected_utility_gain_over_base": 0.012326281517744064, + "selector_regret": 0.027136601507663727, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23290921447763308, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.29761817086705544, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05349260475486517, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.28343513695517897, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23332367249842212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37538333059509166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035802166908979416, + "selected_utility_gain_over_base": -0.06703208014369011, + "selector_regret": 0.001208961009979248, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.268590110511807, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2496823681236075, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.3036913214804951, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17011388186485987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39227093434151067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.023384133353829384, + "selected_utility_gain_over_base": -1.9766158666461706, + "selector_regret": 1.9766158666461706, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.3948568337614319, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08876892179250717, + "selected_utility_gain_over_base": -0.004228569567203522, + "selector_regret": 0.02274090051651001, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5592655513262915, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2525195047503382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.46137979322703987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15205878019332886, + "selected_utility_gain_over_base": 0.102475106716156, + "selector_regret": 1.8479412198066711, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.1768234215359843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11932515699499706, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34125143842352146, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48070996484280953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7407334225672018, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5940825716503084, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3836105423742797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43921922377953376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12788381852052721, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2713166038796752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27481853311866083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21684013213485268, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6097239173416052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21665335322366186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10311308876908776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38798745623031156, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20493799872831076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2762933949412652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09743460718887578, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29487688188584205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29522744379276233, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1808325968136315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19710526958727811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08595256684637552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33681729037438607, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35345287441810663, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18884079158306122, + "selected_utility_gain_over_base": 0.027154862880706787, + "selector_regret": 1.8111592084169388, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.2910706981901193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.5766920271631167, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02248450182378292, + "selected_utility_gain_over_base": 0.020555000170134008, + "selector_regret": 0.016367433592677116, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28546890192574864, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.048270193237023314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.14612092709825528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3319989763272131, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23237107122905468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.1915588212964643, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.30981294473381726, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2531734715082984, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30448894670641713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2722642721291662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2912910141265051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.31737923139011737, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15542103752775793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.46269040126126526, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.051346004009246826, + "selected_utility_gain_over_base": -0.04165148735046387, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.562175027833504, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.5259115214300562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06632735580205917, + "selected_utility_gain_over_base": -1.9336726441979408, + "selector_regret": 0.05877452343702316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4505066498805008, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06990895420312881, + "selected_utility_gain_over_base": 0.020325280725955963, + "selector_regret": 1.9300910457968712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5340001574841613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02059849165380001, + "selected_utility_gain_over_base": -0.08391634561121464, + "selector_regret": 0.09250334836542606, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12054968766555163, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2153900468807876, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4884411626074505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7188844098969164, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5877928871421093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04538704454898834, + "selected_utility_gain_over_base": -1.9546129554510117, + "selector_regret": 1.9546129554510117, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3982631871077599, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13169501516378973, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19620741382881715, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3131801253335776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2250806530775492, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5974323487206175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17691531777381897, + "selected_utility_gain_over_base": -0.013993099331855774, + "selector_regret": 0.05696745216846466, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22061568738801626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17894998490871183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4021201653187239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.18125860724557305, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32995590919912554, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.49032258064516127, + "fallback_rate": 0.5096774193548387, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.44516129032258067, + "selected_success_gain_over_base": 0.16774193548387098, + "selected_utility": 0.9639207383876877, + "selector_regret": 0.2880004003505042, + "success_selector_gap": 0.14193548387096774, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4194444444444444, + "low": 0.2513888888888889, + "mean": 0.33541666666666675, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33541666666666675, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5208888888888888, + "low": 0.05454545454545454, + "mean": 0.2763232323232323, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2763232323232323, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.38666666666666666, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4194444444444444, + "low": 0.2513888888888889, + "mean": 0.33541666666666675, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33541666666666675, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5208888888888888, + "low": 0.05454545454545454, + "mean": 0.2763232323232323, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2763232323232323, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.38666666666666666, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7486111111111112, + "low": 0.5805555555555555, + "mean": 0.6645833333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6645833333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.75, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9454545454545455, + "low": 0.47911111111111104, + "mean": 0.7236767676767677, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236767676767677, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.6133333333333333, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.7222222222222222, + "low": 0.5694444444444444, + "mean": 0.6458333333333334, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4430555555555556, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7206060606060607, + "low": 0.2206060606060606, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3404264536488951, + "low": 0.28329633383431224, + "mean": 0.3113264038608651, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3113264038608651, + "per_seed": { + "1073161778/train0": 0.28355827608242967, + "1073161778/train1": 0.3337636013451598, + "1073161778/train2": 0.29522744379276233, + "1113371261/train0": 0.32662525863909614, + "1113371261/train1": 0.27481853311866083, + "1113371261/train2": 0.3131801253335776, + "1151410346/train0": 0.20532758727946646, + "1151410346/train1": 0.21665335322366186, + "1151410346/train2": 0.22061568738801626, + "1261930456/train0": 0.5566373654374286, + "1261930456/train1": 0.5940825716503084, + "1261930456/train2": 0.5877928871421093, + "1324133122/train0": 0.1953526650841092, + "1324133122/train1": 0.10311308876908776, + "1324133122/train2": 0.17894998490871183, + "1324642411/train0": 0.5244761772076728, + "1324642411/train1": 0.48070996484280953, + "1324642411/train2": 0.4884411626074505, + "1375492786/train0": 0.16043123876023302, + "1375492786/train1": 0.17011388186485987, + "1375492786/train2": 0.15542103752775793, + "141984615/train0": 0.3988789611600871, + "141984615/train1": 0.37538333059509166, + "141984615/train2": 0.30448894670641713, + "1461069713/train0": 0.3050981300072402, + "1461069713/train1": 0.7484074094930177, + "1461069713/train2": 0.2910706981901193, + "1480856810/train0": 0.29221027147425893, + "1480856810/train1": 0.43921922377953376, + "1480856810/train2": 0.29221027147425893, + "1509980664/train0": 0.5870819789979614, + "1509980664/train1": 0.6097239173416052, + "1509980664/train2": 0.5974323487206175, + "1522272686/train0": 0.08863633481483887, + "1522272686/train1": 0.09154681464635614, + "1522272686/train2": 0.08595256684637552, + "1549324805/train0": 0.4534903350362056, + "1549324805/train1": 0.1768234215359843, + "1549324805/train2": 0.5340001574841613, + "1578990937/train0": 0.2255580985362039, + "1578990937/train1": 0.21684013213485268, + "1578990937/train2": 0.2250806530775492, + "1747178030/train0": 0.3352371313866231, + "1747178030/train1": 0.33370414261502274, + "1747178030/train2": 0.33681729037438607, + "1790188787/train0": 0.7256808008161852, + "1790188787/train1": 0.7407334225672018, + "1790188787/train2": 0.7188844098969164, + "1822210471/train0": 0.10685249697353241, + "1822210471/train1": 0.11932515699499706, + "1822210471/train2": 0.12054968766555163, + "1845489013/train0": 0.22822097403519162, + "1845489013/train1": 0.23290921447763308, + "1845489013/train2": 0.23237107122905468, + "1999022574/train0": 0.2830091920285253, + "1999022574/train1": 0.2762933949412652, + "1999022574/train2": 0.32995590919912554, + "2012211287/train0": 0.08991720318616597, + "2012211287/train1": 0.1018745030033766, + "2012211287/train2": 0.09743460718887578, + "2038594191/train0": 0.41980384640853585, + "2038594191/train1": 0.39227093434151067, + "2038594191/train2": 0.41980384640853585, + "2054899126/train0": 0.48682363081170954, + "2054899126/train1": 0.4867067691213679, + "2054899126/train2": 0.4238219648796263, + "2061635888/train0": 0.16138880459399857, + "2061635888/train1": 0.12788381852052721, + "2061635888/train2": 0.13169501516378973, + "211245842/train0": 0.36446911237871227, + "211245842/train1": 0.3818735579959133, + "211245842/train2": 0.3818735579959133, + "278008524/train0": 0.4154810514015457, + "278008524/train1": 0.431062905506948, + "278008524/train2": 0.39125281310385696, + "307068879/train0": 0.2797080480367002, + "307068879/train1": 0.24778606153341362, + "307068879/train2": 0.29487688188584205, + "412102563/train0": 0.20585303592937382, + "412102563/train1": 0.20493799872831076, + "412102563/train2": 0.18125860724557305, + "440577711/train0": 0.1872847414980603, + "440577711/train1": 0.1872847414980603, + "440577711/train2": 0.1872847414980603, + "441046872/train0": 0.20172868695503152, + "441046872/train1": 0.24443771108596596, + "441046872/train2": 0.19710526958727811, + "509430392/train0": 0.39582323274190295, + "509430392/train1": 0.3836105423742797, + "509430392/train2": 0.3982631871077599, + "585904130/train0": 0.26430287849596273, + "585904130/train1": 0.23332367249842212, + "585904130/train2": 0.2531734715082984, + "608016290/train0": 0.18560038716365618, + "608016290/train1": 0.2379598925580104, + "608016290/train2": 0.1808325968136315, + "626486813/train0": 0.41319658430741574, + "626486813/train1": 0.46817656253554407, + "626486813/train2": 0.3841254242297905, + "636680137/train0": 0.5371194860392053, + "636680137/train1": 0.2496823681236075, + "636680137/train2": 0.2912910141265051, + "655884605/train0": 0.30368062376907057, + "655884605/train1": 0.2713166038796752, + "655884605/train2": 0.19620741382881715, + "672328262/train0": 0.042802515015988585, + "672328262/train1": 0.04466194101303914, + "672328262/train2": 0.048270193237023314, + "701667486/train0": 0.3589247640152091, + "701667486/train1": 0.38798745623031156, + "701667486/train2": 0.4021201653187239, + "747260437/train0": 0.28297507531186483, + "747260437/train1": 0.2525195047503382, + "747260437/train2": 0.5259115214300562, + "750448296/train0": 0.4190984327494606, + "750448296/train1": 0.33489946089892014, + "750448296/train2": 0.31678157342373975, + "767747424/train0": 0.28067674068264487, + "767747424/train1": 0.28220702596299274, + "767747424/train2": 0.28169149441007413 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3813259240033606, + "low": 0.16290313710457643, + "mean": 0.27556930577634775, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.27556930577634775, + "per_task": { + "LiftPegUpright-v1": 0.21812074468121995, + "PickCube-v1": 0.08871190543585684, + "PullCube-v1": 0.33025922453409173, + "PushCube-v1": 0.4544794286452365, + "StackCube-v1": 0.2862752255853337 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.34431658455017794, + "low": 0.29466522012600893, + "mean": 0.3189937561563403, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6722222222222223, + "low": 0.5055555555555555, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7430303030303029, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4930555555555556, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4116970675635254, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5246927151765788, + "low": 0.5848668839583795, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.360821089959548, + "low": 1.0729441113071516, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 5.716666666666667, + "low": 4.227083333333334, + "mean": 4.927083333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 4.927083333333333, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 7.0, + "1073161778/train2": 3.0, + "1113371261/train0": 0.0, + "1113371261/train1": 4.0, + "1113371261/train2": 2.0, + "1151410346/train0": 7.0, + "1151410346/train1": 9.0, + "1151410346/train2": 10.0, + "1261930456/train0": 5.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 5.0, + "1324133122/train2": 7.0, + "1324642411/train0": 3.0, + "1324642411/train1": 9.0, + "1324642411/train2": 3.0, + "1375492786/train0": 15.0, + "1375492786/train1": 10.0, + "1375492786/train2": 15.0, + "141984615/train0": 1.0, + "141984615/train1": 6.0, + "141984615/train2": 4.0, + "1461069713/train0": 2.0, + "1461069713/train1": 0.0, + "1461069713/train2": 5.0, + "1480856810/train0": 2.0, + "1480856810/train1": 13.0, + "1480856810/train2": 1.0, + "1509980664/train0": 10.0, + "1509980664/train1": 4.0, + "1509980664/train2": 12.0, + "1522272686/train0": 8.0, + "1522272686/train1": 7.0, + "1522272686/train2": 8.0, + "1549324805/train0": 0.0, + "1549324805/train1": 2.0, + "1549324805/train2": 3.0, + "1578990937/train0": 15.0, + "1578990937/train1": 12.0, + "1578990937/train2": 0.0, + "1747178030/train0": 5.0, + "1747178030/train1": 9.0, + "1747178030/train2": 0.0, + "1790188787/train0": 9.0, + "1790188787/train1": 6.0, + "1790188787/train2": 0.0, + "1822210471/train0": 8.0, + "1822210471/train1": 5.0, + "1822210471/train2": 0.0, + "1845489013/train0": 5.0, + "1845489013/train1": 7.0, + "1845489013/train2": 8.0, + "1999022574/train0": 11.0, + "1999022574/train1": 9.0, + "1999022574/train2": 6.0, + "2012211287/train0": 0.0, + "2012211287/train1": 5.0, + "2012211287/train2": 1.0, + "2038594191/train0": 3.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 5.0, + "2061635888/train1": 6.0, + "2061635888/train2": 4.0, + "211245842/train0": 7.0, + "211245842/train1": 13.0, + "211245842/train2": 0.0, + "278008524/train0": 3.0, + "278008524/train1": 1.0, + "278008524/train2": 11.5, + "307068879/train0": 2.0, + "307068879/train1": 12.0, + "307068879/train2": 0.0, + "412102563/train0": 6.0, + "412102563/train1": 10.0, + "412102563/train2": 3.0, + "440577711/train0": 3.0, + "440577711/train1": 5.0, + "440577711/train2": 5.0, + "441046872/train0": 10.0, + "441046872/train1": 3.0, + "441046872/train2": 0.0, + "509430392/train0": 14.0, + "509430392/train1": 2.0, + "509430392/train2": 1.0, + "585904130/train0": 14.0, + "585904130/train1": 7.0, + "585904130/train2": 4.0, + "608016290/train0": 3.0, + "608016290/train1": 8.0, + "608016290/train2": 15.0, + "626486813/train0": 0.0, + "626486813/train1": 0.5, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 2.0, + "655884605/train2": 8.0, + "672328262/train0": 8.0, + "672328262/train1": 1.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 10.0, + "701667486/train2": 8.0, + "747260437/train0": 0.0, + "747260437/train1": 8.0, + "747260437/train2": 1.0, + "750448296/train0": 4.0, + "750448296/train1": 3.0, + "750448296/train2": 2.0, + "767747424/train0": 7.0, + "767747424/train1": 2.0, + "767747424/train2": 7.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 6.771717171717173, + "low": 4.363555555555555, + "mean": 5.475717171717173, + "n": 5, + "num_samples": 1000 + }, + "mean": 5.475717171717173, + "per_task": { + "LiftPegUpright-v1": 5.0, + "PickCube-v1": 7.666666666666667, + "PullCube-v1": 3.8533333333333335, + "PushCube-v1": 4.555555555555555, + "StackCube-v1": 6.303030303030303 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 5.409722222222222, + "low": 4.020833333333333, + "mean": 4.701388888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3875, + "low": 0.21875, + "mean": 0.3020833333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3020833333333333, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.29943434343434344, + "low": 0.10060606060606062, + "mean": 0.2174949494949495, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2174949494949495, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.30666666666666664, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2152777777777778, + "mean": 0.2847222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.041666666666666664, + "low": -0.05416666666666667, + "mean": -0.004166666666666667, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.004166666666666667, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.0, + "2038594191/train1": -1.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.06666666666666667, + "low": -0.02593939393939394, + "mean": 0.017616161616161613, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.017616161616161613, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.05333333333333334, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.034722222222222224, + "low": -0.05555555555555555, + "mean": -0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9239329974577736, + "low": 0.6214197847254884, + "mean": 0.7678461809070238, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7678461809070238, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 0.04538704454898834, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.061532557010650635, + "1324642411/train1": 0.1280023753643036, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.0042244791984558105, + "141984615/train1": 0.035802166908979416, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.009687979705631733, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.20681259036064148, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 2.0, + "1509980664/train1": 0.21804393827915192, + "1509980664/train2": 0.8189421892166138, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.02059849165380001, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.14965972304344177, + "1790188787/train1": 0.13741976022720337, + "1790188787/train2": 2.0, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 0.05387979745864868, + "2038594191/train1": 0.023384133353829384, + "2038594191/train2": 2.0, + "2054899126/train0": 1.052448581904173, + "2054899126/train1": 1.0664558708667755, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.11833827197551727, + "211245842/train1": 0.17156320810317993, + "211245842/train2": 0.17691531777381897, + "278008524/train0": 1.0485825426876545, + "278008524/train1": 0.1001356840133667, + "278008524/train2": 0.052897630259394646, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.12097799777984619, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.005386819248087704, + "626486813/train1": 0.00787139916792512, + "626486813/train2": 0.015375842805951834, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 0.06632735580205917, + "750448296/train0": 0.09317883538703124, + "750448296/train1": 0.11618362615505855, + "750448296/train2": 0.1037093202273051, + "767747424/train0": 0.5386653137393296, + "767747424/train1": 0.5581304961815476, + "767747424/train2": 0.5799880577251315 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8515048929266256, + "low": 0.6142033413514081, + "mean": 0.7215833358311028, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7215833358311028, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.666174189314867, + "PushCube-v1": 0.6184621560904715, + "StackCube-v1": 0.7894506924080126 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8554598261252977, + "low": 0.5858313379317729, + "mean": 0.7162999392102292, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.06609533377680539, + "low": -0.11685185874133215, + "mean": -0.02227188139077043, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.02227188139077043, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.9546129554510117, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -0.044937968254089355, + "1324642411/train1": 0.0215318500995636, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.09860976785421371, + "141984615/train1": -0.06703208014369011, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": -0.08421508502215147, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.11977851390838623, + "1480856810/train2": 0.0, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": -0.6153660863637924, + "1509980664/train2": -0.014467835426330566, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": 0.0, + "1549324805/train2": -0.08391634561121464, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.024892255663871765, + "1790188787/train1": 0.012652292847633362, + "1790188787/train2": 1.87523253262043, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.9461202025413513, + "2038594191/train1": -1.9766158666461706, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9475514180958271, + "2054899126/train1": -0.9335441291332245, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": -0.07257014513015747, + "211245842/train1": -0.019345209002494812, + "211245842/train2": -0.013993099331855774, + "278008524/train0": 1.0058475527912378, + "278008524/train1": 0.057400694116950035, + "278008524/train2": 0.010162640362977982, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.12097777426242828, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0002884765272028744, + "626486813/train1": 0.0027730564470402896, + "626486813/train2": 0.010277500085067004, + "636680137/train0": -0.027042903006076813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.9336726441979408, + "750448296/train0": -0.024414313957095146, + "750448296/train1": -0.0014095231890678406, + "750448296/train2": -0.013883829116821289, + "767747424/train0": -0.03453402826562524, + "767747424/train1": -0.015068845823407173, + "767747424/train2": 0.006788715720176697 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1356347863872846, + "low": -0.06278656556877787, + "mean": 0.026023538657087796, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.026023538657087796, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.10899627190859368, + "PushCube-v1": 0.226057977312141, + "StackCube-v1": 0.013055987881891655 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.05458070211009019, + "low": -0.12055642578343395, + "mean": -0.025519647232108075, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6801490995260085, + "low": 0.40305140065320294, + "mean": 0.5353515408005073, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5353515408005073, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.9546129554510117, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.08252748847007751, + "1324642411/train1": 1.8719976246356964, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.010953426361083984, + "141984615/train1": 0.001208961009979248, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.00689703319221735, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.7931874096393585, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 0.0, + "1509980664/train1": 1.781956061720848, + "1509980664/train2": 1.1810578107833862, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.09250334836542606, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.8503402769565582, + "1790188787/train1": 1.8625802397727966, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 1.9461202025413513, + "2038594191/train1": 1.9766158666461706, + "2038594191/train2": 0.0, + "2054899126/train0": 0.024834569543600082, + "2054899126/train1": 0.9335441291332245, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8816617280244827, + "211245842/train1": 0.04139396548271179, + "211245842/train2": 0.05696745216846466, + "278008524/train0": 0.02477378025650978, + "278008524/train1": 0.9375389106571674, + "278008524/train2": 1.0830512810498476, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 1.3000448346138, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8790220022201538, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.03930865635629743, + "626486813/train1": 0.027956475503742695, + "626486813/train2": 0.029523412231355906, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.6498691290616989, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.05877452343702316, + "750448296/train0": 0.6594814925144116, + "750448296/train1": 1.2543196479479473, + "750448296/train2": 1.246739347775777, + "767747424/train0": 0.5124941510148346, + "767747424/train1": 0.5076578482985497, + "767747424/train2": 0.48825800977647305 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8738215418463066, + "low": 0.17712627903981643, + "mean": 0.5098326246412377, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5098326246412377, + "per_task": { + "LiftPegUpright-v1": 0.5146939317385356, + "PickCube-v1": 0.0, + "PullCube-v1": 0.49621246912516653, + "PushCube-v1": 1.1673192588819399, + "StackCube-v1": 0.37093746346054657 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.6691991107056512, + "low": 0.41660342318290433, + "mean": 0.5429793549620727, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38333333333333336, + "low": 0.225, + "mean": 0.29930555555555555, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.29930555555555555, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4680000000000001, + "low": 0.12242424242424244, + "mean": 0.2953131313131313, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2953131313131313, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.25333333333333335, + "PushCube-v1": 0.6111111111111112, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2222222222222222, + "mean": 0.2986111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30416666666666664, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7191919191919192, + "low": 0.13511111111111113, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2638888888888889, + "low": 0.1388888888888889, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5434736852068454, + "low": 0.2965679521788843, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.330930626693398, + "low": 0.24626440485649642, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.485598717091812, + "low": 0.272009730946997, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.34731881885037813, + "low": 0.34731881885037813, + "mean": 0.34731881885037813, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.34731881885037813, + "per_seed": { + "1073161778/train0": 0.3473188188503786, + "1073161778/train1": 0.3473188188503786, + "1073161778/train2": 0.3473188188503786, + "1113371261/train0": 0.3473188188503786, + "1113371261/train1": 0.3473188188503786, + "1113371261/train2": 0.3473188188503786, + "1151410346/train0": 0.3473188188503786, + "1151410346/train1": 0.3473188188503786, + "1151410346/train2": 0.3473188188503786, + "1261930456/train0": 0.3473188188503786, + "1261930456/train1": 0.3473188188503786, + "1261930456/train2": 0.3473188188503786, + "1324133122/train0": 0.3473188188503786, + "1324133122/train1": 0.3473188188503786, + "1324133122/train2": 0.3473188188503786, + "1324642411/train0": 0.3473188188503786, + "1324642411/train1": 0.3473188188503786, + "1324642411/train2": 0.3473188188503786, + "1375492786/train0": 0.3473188188503786, + "1375492786/train1": 0.3473188188503786, + "1375492786/train2": 0.3473188188503786, + "141984615/train0": 0.3473188188503786, + "141984615/train1": 0.3473188188503786, + "141984615/train2": 0.3473188188503786, + "1461069713/train0": 0.3473188188503786, + "1461069713/train1": 0.3473188188503786, + "1461069713/train2": 0.3473188188503786, + "1480856810/train0": 0.3473188188503786, + "1480856810/train1": 0.3473188188503786, + "1480856810/train2": 0.3473188188503786, + "1509980664/train0": 0.3473188188503786, + "1509980664/train1": 0.3473188188503786, + "1509980664/train2": 0.3473188188503786, + "1522272686/train0": 0.3473188188503786, + "1522272686/train1": 0.3473188188503786, + "1522272686/train2": 0.3473188188503786, + "1549324805/train0": 0.3473188188503786, + "1549324805/train1": 0.3473188188503786, + "1549324805/train2": 0.3473188188503786, + "1578990937/train0": 0.3473188188503786, + "1578990937/train1": 0.3473188188503786, + "1578990937/train2": 0.3473188188503786, + "1747178030/train0": 0.3473188188503786, + "1747178030/train1": 0.3473188188503786, + "1747178030/train2": 0.3473188188503786, + "1790188787/train0": 0.3473188188503786, + "1790188787/train1": 0.3473188188503786, + "1790188787/train2": 0.3473188188503786, + "1822210471/train0": 0.3473188188503786, + "1822210471/train1": 0.3473188188503786, + "1822210471/train2": 0.3473188188503786, + "1845489013/train0": 0.3473188188503786, + "1845489013/train1": 0.3473188188503786, + "1845489013/train2": 0.3473188188503786, + "1999022574/train0": 0.3473188188503786, + "1999022574/train1": 0.3473188188503786, + "1999022574/train2": 0.3473188188503786, + "2012211287/train0": 0.3473188188503786, + "2012211287/train1": 0.3473188188503786, + "2012211287/train2": 0.3473188188503786, + "2038594191/train0": 0.3473188188503786, + "2038594191/train1": 0.3473188188503786, + "2038594191/train2": 0.3473188188503786, + "2054899126/train0": 0.3473188188503786, + "2054899126/train1": 0.3473188188503786, + "2054899126/train2": 0.3473188188503786, + "2061635888/train0": 0.3473188188503786, + "2061635888/train1": 0.3473188188503786, + "2061635888/train2": 0.3473188188503786, + "211245842/train0": 0.3473188188503786, + "211245842/train1": 0.3473188188503786, + "211245842/train2": 0.3473188188503786, + "278008524/train0": 0.3473188188503786, + "278008524/train1": 0.3473188188503786, + "278008524/train2": 0.3473188188503786, + "307068879/train0": 0.3473188188503786, + "307068879/train1": 0.3473188188503786, + "307068879/train2": 0.3473188188503786, + "412102563/train0": 0.3473188188503786, + "412102563/train1": 0.3473188188503786, + "412102563/train2": 0.3473188188503786, + "440577711/train0": 0.3473188188503786, + "440577711/train1": 0.3473188188503786, + "440577711/train2": 0.3473188188503786, + "441046872/train0": 0.3473188188503786, + "441046872/train1": 0.3473188188503786, + "441046872/train2": 0.3473188188503786, + "509430392/train0": 0.3473188188503786, + "509430392/train1": 0.3473188188503786, + "509430392/train2": 0.3473188188503786, + "585904130/train0": 0.3473188188503786, + "585904130/train1": 0.3473188188503786, + "585904130/train2": 0.3473188188503786, + "608016290/train0": 0.3473188188503786, + "608016290/train1": 0.3473188188503786, + "608016290/train2": 0.3473188188503786, + "626486813/train0": 0.3473188188503786, + "626486813/train1": 0.3473188188503786, + "626486813/train2": 0.3473188188503786, + "636680137/train0": 0.3473188188503786, + "636680137/train1": 0.3473188188503786, + "636680137/train2": 0.3473188188503786, + "655884605/train0": 0.3473188188503786, + "655884605/train1": 0.3473188188503786, + "655884605/train2": 0.3473188188503786, + "672328262/train0": 0.3473188188503786, + "672328262/train1": 0.3473188188503786, + "672328262/train2": 0.3473188188503786, + "701667486/train0": 0.3473188188503786, + "701667486/train1": 0.3473188188503786, + "701667486/train2": 0.3473188188503786, + "747260437/train0": 0.3473188188503786, + "747260437/train1": 0.3473188188503786, + "747260437/train2": 0.3473188188503786, + "750448296/train0": 0.3473188188503786, + "750448296/train1": 0.3473188188503786, + "750448296/train2": 0.3473188188503786, + "767747424/train0": 0.3473188188503786, + "767747424/train1": 0.3473188188503786, + "767747424/train2": 0.3473188188503786 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.34731881885037874, + "low": 0.34731881885037846, + "mean": 0.3473188188503786, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3473188188503786, + "per_task": { + "LiftPegUpright-v1": 0.34731881885037846, + "PickCube-v1": 0.3473188188503786, + "PullCube-v1": 0.34731881885037885, + "PushCube-v1": 0.3473188188503784, + "StackCube-v1": 0.3473188188503786 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.3473188188503777, + "low": 0.3473188188503777, + "mean": 0.3473188188503777, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "positive_margin", + "tau": 0.3473188188503786 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..0bf7f4dae4c0ac2069e096d6ba0e5ffe11eb84c7 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28355827608242967, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3337636013451598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29522744379276233, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32662525863909614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.3473188188503786 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27481853311866083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.3473188188503786 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3131801253335776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.3473188188503786 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20532758727946646, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.3473188188503786 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21665335322366186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22061568738801626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.3473188188503786 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5566373654374286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5940825716503084, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5877928871421093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04538704454898834, + "selected_utility_gain_over_base": -1.9546129554510117, + "selector_regret": 1.9546129554510117, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1953526650841092, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.3473188188503786 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10311308876908776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.3473188188503786 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17894998490871183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.3473188188503786 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244761772076728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.061532557010650635, + "selected_utility_gain_over_base": -0.044937968254089355, + "selector_regret": 0.08252748847007751, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.3473188188503786 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48070996484280953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4884411626074505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16043123876023302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17011388186485987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15542103752775793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3988789611600871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0042244791984558105, + "selected_utility_gain_over_base": -0.09860976785421371, + "selector_regret": 0.010953426361083984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.3473188188503786 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37538333059509166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035802166908979416, + "selected_utility_gain_over_base": -0.06703208014369011, + "selector_regret": 0.001208961009979248, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.3473188188503786 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30448894670641713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.3473188188503786 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3050981300072402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.3473188188503786 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.7484074094930177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.3473188188503786 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.2910706981901193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.3473188188503786 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43921922377953376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5870819789979614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6097239173416052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5974323487206175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08863633481483887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.3473188188503786 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09154681464635614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.3473188188503786 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08595256684637552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.3473188188503786 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.4534903350362056, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.3473188188503786 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.1768234215359843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.3473188188503786 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5340001574841613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02059849165380001, + "selected_utility_gain_over_base": -0.08391634561121464, + "selector_regret": 0.09250334836542606, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.3473188188503786 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2255580985362039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21684013213485268, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2250806530775492, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3352371313866231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33370414261502274, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33681729037438607, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7256808008161852, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14965972304344177, + "selected_utility_gain_over_base": 0.024892255663871765, + "selector_regret": 1.8503402769565582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7407334225672018, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7188844098969164, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10685249697353241, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11932515699499706, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12054968766555163, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22822097403519162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23290921447763308, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23237107122905468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2830091920285253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2762933949412652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32995590919912554, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.08991720318616597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1018745030033766, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09743460718887578, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05387979745864868, + "selected_utility_gain_over_base": -1.9461202025413513, + "selector_regret": 1.9461202025413513, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39227093434151067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.023384133353829384, + "selected_utility_gain_over_base": -1.9766158666461706, + "selector_regret": 1.9766158666461706, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.48682363081170954, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.052448581904173, + "selected_utility_gain_over_base": -0.9475514180958271, + "selector_regret": 0.024834569543600082, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.3473188188503786 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4867067691213679, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0664558708667755, + "selected_utility_gain_over_base": -0.9335441291332245, + "selector_regret": 0.9335441291332245, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4238219648796263, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16138880459399857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.3473188188503786 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12788381852052721, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.3473188188503786 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13169501516378973, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.3473188188503786 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36446911237871227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11833827197551727, + "selected_utility_gain_over_base": -0.07257014513015747, + "selector_regret": 1.8816617280244827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.3473188188503786 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17691531777381897, + "selected_utility_gain_over_base": -0.013993099331855774, + "selector_regret": 0.05696745216846466, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.3473188188503786 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.4154810514015457, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0485825426876545, + "selected_utility_gain_over_base": 1.0058475527912378, + "selector_regret": 0.02477378025650978, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.3473188188503786 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.431062905506948, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1001356840133667, + "selected_utility_gain_over_base": 0.057400694116950035, + "selector_regret": 0.9375389106571674, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.3473188188503786 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.39125281310385696, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 11.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.052897630259394646, + "selected_utility_gain_over_base": 0.010162640362977982, + "selector_regret": 1.0830512810498476, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2797080480367002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.3473188188503786 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24778606153341362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.3473188188503786 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29487688188584205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.3473188188503786 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20585303592937382, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20493799872831076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.18125860724557305, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20172868695503152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24443771108596596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.3473188188503786 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19710526958727811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.3473188188503786 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.39582323274190295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3836105423742797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3982631871077599, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.3473188188503786 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26430287849596273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23332367249842212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2531734715082984, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18560038716365618, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2379598925580104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1808325968136315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.3473188188503786 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.41319658430741574, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.005386819248087704, + "selected_utility_gain_over_base": 0.0002884765272028744, + "selector_regret": 0.03930865635629743, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.3473188188503786 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.46817656253554407, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00787139916792512, + "selected_utility_gain_over_base": 0.0027730564470402896, + "selector_regret": 0.027956475503742695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.3473188188503786 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.3841254242297905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015375842805951834, + "selected_utility_gain_over_base": 0.010277500085067004, + "selector_regret": 0.029523412231355906, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.3473188188503786 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371194860392053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.3473188188503786 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2496823681236075, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.3473188188503786 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2912910141265051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.3473188188503786 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30368062376907057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.3473188188503786 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2713166038796752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.3473188188503786 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19620741382881715, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.3473188188503786 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042802515015988585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04466194101303914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.048270193237023314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3589247640152091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38798745623031156, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4021201653187239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.28297507531186483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.3473188188503786 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2525195047503382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.3473188188503786 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.5259115214300562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06632735580205917, + "selected_utility_gain_over_base": -1.9336726441979408, + "selector_regret": 0.05877452343702316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.3473188188503786 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.4190984327494606, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09317883538703124, + "selected_utility_gain_over_base": -0.024414313957095146, + "selector_regret": 0.6594814925144116, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.3473188188503786 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.33489946089892014, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11618362615505855, + "selected_utility_gain_over_base": -0.0014095231890678406, + "selector_regret": 1.2543196479479473, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.3473188188503786 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.31678157342373975, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1037093202273051, + "selected_utility_gain_over_base": -0.013883829116821289, + "selector_regret": 1.246739347775777, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.3473188188503786 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28067674068264487, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 7.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5386653137393296, + "selected_utility_gain_over_base": -0.03453402826562524, + "selector_regret": 0.5124941510148346, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.3473188188503786 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28220702596299274, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 2.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5581304961815476, + "selected_utility_gain_over_base": -0.015068845823407173, + "selector_regret": 0.5076578482985497, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.3473188188503786 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28169149441007413, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 7.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5799880577251315, + "selected_utility_gain_over_base": 0.006788715720176697, + "selector_regret": 0.48825800977647305, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.3473188188503786 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..a11dc53387354f6dc7e42837a1c83efaa31679e1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "predicted_margin": 0.21812074468121995, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 5.0, + "selected_success": 0.2, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.9498705387115478, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5146939317385356, + "success_selector_gap": 0.4, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.34731881885037846 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08871190543585684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 7.666666666666667, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.3473188188503786 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.38666666666666666, + "execute_generated": 0.38666666666666666, + "fallback_rate": 0.6133333333333333, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.33025922453409173, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 3.8533333333333335, + "selected_success": 0.30666666666666664, + "selected_success_gain_over_base": -0.05333333333333334, + "selected_utility": 0.666174189314867, + "selected_utility_gain_over_base": -0.10899627190859368, + "selector_regret": 0.49621246912516653, + "success_selector_gap": 0.25333333333333335, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.34731881885037885 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.7222222222222222, + "execute_generated": 0.7222222222222222, + "fallback_rate": 0.2777777777777778, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "predicted_margin": 0.4544794286452365, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 4.555555555555555, + "selected_success": 0.2777777777777778, + "selected_success_gain_over_base": 0.1111111111111111, + "selected_utility": 0.6184621560904715, + "selected_utility_gain_over_base": 0.226057977312141, + "selector_regret": 1.1673192588819399, + "success_selector_gap": 0.6111111111111112, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.3473188188503784 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.2727272727272727, + "execute_generated": 0.2727272727272727, + "fallback_rate": 0.7272727272727273, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "predicted_margin": 0.2862752255853337, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 6.303030303030303, + "selected_success": 0.30303030303030304, + "selected_success_gain_over_base": 0.030303030303030304, + "selected_utility": 0.7894506924080126, + "selected_utility_gain_over_base": 0.013055987881891655, + "selector_regret": 0.37093746346054657, + "success_selector_gap": 0.21212121212121213, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.3473188188503786 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..82ded9aeb0b54d481359946a565e9d403428534e --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.347319` +Feature set: `chart_compat` +Target: `positive_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4549 | 0.5451 | 0.2780 | 0.4621 | 0.5343 | 0.5343 | 0.1733 | 0.1264 | +| selection | 0.4903 | 0.5097 | 0.2774 | 0.4452 | 0.5419 | 0.5677 | 0.1806 | 0.1419 | +| eval | 0.3542 | 0.6458 | 0.2917 | 0.2847 | 0.5694 | 0.5486 | 0.2014 | 0.2986 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..2706c3e0547971eab692012771ffa1c44982d2e5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,20 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + }, + "source_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..a3d945b1021ea4fa647f2f4f5760bbc06c826384 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.3542 & 0.6458 & 0.2917 & 0.2847 & 0.5694 & 0.5486 & 0.2014 & 0.2986 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..601d7f3b02f91084eb580b305aaa600dddb73f9f --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0} +tau=0.347319 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..751cc573212f88f9ae454306af36f780ce05a987 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test --feature-set chart_compat --selector-chart-feature-mode base_context_obs --target success_weighted_margin --k 16 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c87581b11accf1a4f062d3fd43d9455de6271683 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,15 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_compat +k: 16 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test +seed: 0 +selection_frac: 0.35 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +target: success_weighted_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c001552e076e38f762c1980a9ac2fdd590af6dc4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..06b1ba509f2833a3e3390e36e6dae4aba487d8e6 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,7470 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.3541666666666667, + "fallback_rate": 0.6458333333333334, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.2847222222222222, + "selected_success_gain_over_base": -0.006944444444444444, + "selected_utility": 0.7162999392102292, + "selector_regret": 0.5429793549620727, + "success_selector_gap": 0.2986111111111111, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "chart_compat", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4548736462093863, + "fallback_rate": 0.5451263537906137, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.5342960288808665, + "proposal_oracle_success": 0.5342960288808665, + "proposal_oracle_utility": 1.1336619752116286, + "selected_success": 0.4620938628158845, + "selected_success_gain_over_base": 0.18411552346570398, + "selected_utility": 1.0263910738352875, + "selector_regret": 0.2268675415909476, + "success_selector_gap": 0.1263537906137184, + "success_support_gap": 0.17328519855595667, + "support_gap": 0.3521794904517649 + }, + "k": 16, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.49032258064516127, + "fallback_rate": 0.5096774193548387, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.44516129032258067, + "selected_success_gain_over_base": 0.16774193548387098, + "selected_utility": 0.9639207383876877, + "selector_regret": 0.2880004003505042, + "success_selector_gap": 0.14193548387096774, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "num_fit_candidates": 4175, + "num_fit_rows": 277, + "num_selection_candidates": 2335, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.08991720318616597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2797080480367002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28355827608242967, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18560038716365618, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20172868695503152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08863633481483887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3352371313866231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4110900404587235, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.023549815639853477, + "selected_utility_gain_over_base": -0.13813611306250095, + "selector_regret": 1.9764501843601465, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3050981300072402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.6362267958435123, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0025064547080546618, + "selected_utility_gain_over_base": 0.0005769530544057488, + "selector_regret": 0.007676919689401984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.507461874569938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1048971638083458, + "selected_utility_gain_over_base": -1.8951028361916542, + "selector_regret": 0.049669139087200165, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042802515015988585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.17492469965849014, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.35986959125133827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09716508537530899, + "selected_utility_gain_over_base": 0.061278779059648514, + "selector_regret": 0.04954756051301956, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22822097403519162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.19016637277131918, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.26403852161187397, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26430287849596273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3988789611600871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0042244791984558105, + "selected_utility_gain_over_base": -0.09860976785421371, + "selector_regret": 0.010953426361083984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34069303977256765, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371194860392053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.2726537010014917, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16043123876023302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05387979745864868, + "selected_utility_gain_over_base": -1.9461202025413513, + "selector_regret": 1.9461202025413513, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.615562919823872, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019754549488425255, + "selected_utility_gain_over_base": -0.07324294187128544, + "selector_regret": 0.02163984440267086, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46618538705348106, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.28297507531186483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.471092511551753, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.4534903350362056, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10685249697353241, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3010393386519421, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244761772076728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.061532557010650635, + "selected_utility_gain_over_base": -0.044937968254089355, + "selector_regret": 0.08252748847007751, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7256808008161852, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14965972304344177, + "selected_utility_gain_over_base": 0.024892255663871765, + "selector_regret": 1.8503402769565582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5566373654374286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.39582323274190295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16138880459399857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30368062376907057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32662525863909614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2255580985362039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5870819789979614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36446911237871227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11833827197551727, + "selected_utility_gain_over_base": -0.07257014513015747, + "selector_regret": 1.8816617280244827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20532758727946646, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1953526650841092, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3589247640152091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20585303592937382, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2830091920285253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1018745030033766, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24778606153341362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3337636013451598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2379598925580104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24443771108596596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09154681464635614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33370414261502274, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3705938518118879, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10141054540872574, + "selected_utility_gain_over_base": -0.06027538329362869, + "selector_regret": 1.8985894545912743, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.7484074094930177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.6387349542040327, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007475614547729492, + "selected_utility_gain_over_base": 0.005546112894080579, + "selector_regret": 0.00242034625262022, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41414798691644433, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13291174173355103, + "selected_utility_gain_over_base": -1.867088258266449, + "selector_regret": 1.867088258266449, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04466194101303914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.17110779360440906, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.4007460177868561, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04821258783340454, + "selected_utility_gain_over_base": 0.012326281517744064, + "selector_regret": 0.027136601507663727, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23290921447763308, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.29761817086705544, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05349260475486517, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.28343513695517897, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23332367249842212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37538333059509166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035802166908979416, + "selected_utility_gain_over_base": -0.06703208014369011, + "selector_regret": 0.001208961009979248, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.268590110511807, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2496823681236075, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.3036913214804951, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17011388186485987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39227093434151067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.023384133353829384, + "selected_utility_gain_over_base": -1.9766158666461706, + "selector_regret": 1.9766158666461706, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.3948568337614319, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08876892179250717, + "selected_utility_gain_over_base": -0.004228569567203522, + "selector_regret": 0.02274090051651001, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5592655513262915, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2525195047503382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.46137979322703987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15205878019332886, + "selected_utility_gain_over_base": 0.102475106716156, + "selector_regret": 1.8479412198066711, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.1768234215359843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11932515699499706, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34125143842352146, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48070996484280953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7407334225672018, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5940825716503084, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3836105423742797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43921922377953376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12788381852052721, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2713166038796752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27481853311866083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21684013213485268, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6097239173416052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21665335322366186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10311308876908776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38798745623031156, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20493799872831076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2762933949412652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09743460718887578, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29487688188584205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29522744379276233, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1808325968136315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19710526958727811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08595256684637552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33681729037438607, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35345287441810663, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18884079158306122, + "selected_utility_gain_over_base": 0.027154862880706787, + "selector_regret": 1.8111592084169388, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.2910706981901193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.5766920271631167, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02248450182378292, + "selected_utility_gain_over_base": 0.020555000170134008, + "selector_regret": 0.016367433592677116, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28546890192574864, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.048270193237023314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.14612092709825528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3319989763272131, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23237107122905468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.1915588212964643, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.30981294473381726, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2531734715082984, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30448894670641713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2722642721291662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2912910141265051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.31737923139011737, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15542103752775793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.46269040126126526, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.051346004009246826, + "selected_utility_gain_over_base": -0.04165148735046387, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.562175027833504, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.5259115214300562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06632735580205917, + "selected_utility_gain_over_base": -1.9336726441979408, + "selector_regret": 0.05877452343702316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4505066498805008, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06990895420312881, + "selected_utility_gain_over_base": 0.020325280725955963, + "selector_regret": 1.9300910457968712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5340001574841613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02059849165380001, + "selected_utility_gain_over_base": -0.08391634561121464, + "selector_regret": 0.09250334836542606, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12054968766555163, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2153900468807876, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4884411626074505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7188844098969164, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5877928871421093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04538704454898834, + "selected_utility_gain_over_base": -1.9546129554510117, + "selector_regret": 1.9546129554510117, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3982631871077599, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13169501516378973, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19620741382881715, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3131801253335776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2250806530775492, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5974323487206175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17691531777381897, + "selected_utility_gain_over_base": -0.013993099331855774, + "selector_regret": 0.05696745216846466, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22061568738801626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17894998490871183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4021201653187239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.18125860724557305, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32995590919912554, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.49032258064516127, + "fallback_rate": 0.5096774193548387, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.44516129032258067, + "selected_success_gain_over_base": 0.16774193548387098, + "selected_utility": 0.9639207383876877, + "selector_regret": 0.2880004003505042, + "success_selector_gap": 0.14193548387096774, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4194444444444444, + "low": 0.2513888888888889, + "mean": 0.33541666666666675, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33541666666666675, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5208888888888888, + "low": 0.05454545454545454, + "mean": 0.2763232323232323, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2763232323232323, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.38666666666666666, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4194444444444444, + "low": 0.2513888888888889, + "mean": 0.33541666666666675, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33541666666666675, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5208888888888888, + "low": 0.05454545454545454, + "mean": 0.2763232323232323, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2763232323232323, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.38666666666666666, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7486111111111112, + "low": 0.5805555555555555, + "mean": 0.6645833333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6645833333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.75, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9454545454545455, + "low": 0.47911111111111104, + "mean": 0.7236767676767677, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236767676767677, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.6133333333333333, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.7222222222222222, + "low": 0.5694444444444444, + "mean": 0.6458333333333334, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4430555555555556, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7206060606060607, + "low": 0.2206060606060606, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3404264536488951, + "low": 0.28329633383431224, + "mean": 0.3113264038608651, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3113264038608651, + "per_seed": { + "1073161778/train0": 0.28355827608242967, + "1073161778/train1": 0.3337636013451598, + "1073161778/train2": 0.29522744379276233, + "1113371261/train0": 0.32662525863909614, + "1113371261/train1": 0.27481853311866083, + "1113371261/train2": 0.3131801253335776, + "1151410346/train0": 0.20532758727946646, + "1151410346/train1": 0.21665335322366186, + "1151410346/train2": 0.22061568738801626, + "1261930456/train0": 0.5566373654374286, + "1261930456/train1": 0.5940825716503084, + "1261930456/train2": 0.5877928871421093, + "1324133122/train0": 0.1953526650841092, + "1324133122/train1": 0.10311308876908776, + "1324133122/train2": 0.17894998490871183, + "1324642411/train0": 0.5244761772076728, + "1324642411/train1": 0.48070996484280953, + "1324642411/train2": 0.4884411626074505, + "1375492786/train0": 0.16043123876023302, + "1375492786/train1": 0.17011388186485987, + "1375492786/train2": 0.15542103752775793, + "141984615/train0": 0.3988789611600871, + "141984615/train1": 0.37538333059509166, + "141984615/train2": 0.30448894670641713, + "1461069713/train0": 0.3050981300072402, + "1461069713/train1": 0.7484074094930177, + "1461069713/train2": 0.2910706981901193, + "1480856810/train0": 0.29221027147425893, + "1480856810/train1": 0.43921922377953376, + "1480856810/train2": 0.29221027147425893, + "1509980664/train0": 0.5870819789979614, + "1509980664/train1": 0.6097239173416052, + "1509980664/train2": 0.5974323487206175, + "1522272686/train0": 0.08863633481483887, + "1522272686/train1": 0.09154681464635614, + "1522272686/train2": 0.08595256684637552, + "1549324805/train0": 0.4534903350362056, + "1549324805/train1": 0.1768234215359843, + "1549324805/train2": 0.5340001574841613, + "1578990937/train0": 0.2255580985362039, + "1578990937/train1": 0.21684013213485268, + "1578990937/train2": 0.2250806530775492, + "1747178030/train0": 0.3352371313866231, + "1747178030/train1": 0.33370414261502274, + "1747178030/train2": 0.33681729037438607, + "1790188787/train0": 0.7256808008161852, + "1790188787/train1": 0.7407334225672018, + "1790188787/train2": 0.7188844098969164, + "1822210471/train0": 0.10685249697353241, + "1822210471/train1": 0.11932515699499706, + "1822210471/train2": 0.12054968766555163, + "1845489013/train0": 0.22822097403519162, + "1845489013/train1": 0.23290921447763308, + "1845489013/train2": 0.23237107122905468, + "1999022574/train0": 0.2830091920285253, + "1999022574/train1": 0.2762933949412652, + "1999022574/train2": 0.32995590919912554, + "2012211287/train0": 0.08991720318616597, + "2012211287/train1": 0.1018745030033766, + "2012211287/train2": 0.09743460718887578, + "2038594191/train0": 0.41980384640853585, + "2038594191/train1": 0.39227093434151067, + "2038594191/train2": 0.41980384640853585, + "2054899126/train0": 0.48682363081170954, + "2054899126/train1": 0.4867067691213679, + "2054899126/train2": 0.4238219648796263, + "2061635888/train0": 0.16138880459399857, + "2061635888/train1": 0.12788381852052721, + "2061635888/train2": 0.13169501516378973, + "211245842/train0": 0.36446911237871227, + "211245842/train1": 0.3818735579959133, + "211245842/train2": 0.3818735579959133, + "278008524/train0": 0.4154810514015457, + "278008524/train1": 0.431062905506948, + "278008524/train2": 0.39125281310385696, + "307068879/train0": 0.2797080480367002, + "307068879/train1": 0.24778606153341362, + "307068879/train2": 0.29487688188584205, + "412102563/train0": 0.20585303592937382, + "412102563/train1": 0.20493799872831076, + "412102563/train2": 0.18125860724557305, + "440577711/train0": 0.1872847414980603, + "440577711/train1": 0.1872847414980603, + "440577711/train2": 0.1872847414980603, + "441046872/train0": 0.20172868695503152, + "441046872/train1": 0.24443771108596596, + "441046872/train2": 0.19710526958727811, + "509430392/train0": 0.39582323274190295, + "509430392/train1": 0.3836105423742797, + "509430392/train2": 0.3982631871077599, + "585904130/train0": 0.26430287849596273, + "585904130/train1": 0.23332367249842212, + "585904130/train2": 0.2531734715082984, + "608016290/train0": 0.18560038716365618, + "608016290/train1": 0.2379598925580104, + "608016290/train2": 0.1808325968136315, + "626486813/train0": 0.41319658430741574, + "626486813/train1": 0.46817656253554407, + "626486813/train2": 0.3841254242297905, + "636680137/train0": 0.5371194860392053, + "636680137/train1": 0.2496823681236075, + "636680137/train2": 0.2912910141265051, + "655884605/train0": 0.30368062376907057, + "655884605/train1": 0.2713166038796752, + "655884605/train2": 0.19620741382881715, + "672328262/train0": 0.042802515015988585, + "672328262/train1": 0.04466194101303914, + "672328262/train2": 0.048270193237023314, + "701667486/train0": 0.3589247640152091, + "701667486/train1": 0.38798745623031156, + "701667486/train2": 0.4021201653187239, + "747260437/train0": 0.28297507531186483, + "747260437/train1": 0.2525195047503382, + "747260437/train2": 0.5259115214300562, + "750448296/train0": 0.4190984327494606, + "750448296/train1": 0.33489946089892014, + "750448296/train2": 0.31678157342373975, + "767747424/train0": 0.28067674068264487, + "767747424/train1": 0.28220702596299274, + "767747424/train2": 0.28169149441007413 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3813259240033606, + "low": 0.16290313710457643, + "mean": 0.27556930577634775, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.27556930577634775, + "per_task": { + "LiftPegUpright-v1": 0.21812074468121995, + "PickCube-v1": 0.08871190543585684, + "PullCube-v1": 0.33025922453409173, + "PushCube-v1": 0.4544794286452365, + "StackCube-v1": 0.2862752255853337 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.34431658455017794, + "low": 0.29466522012600893, + "mean": 0.3189937561563403, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6722222222222223, + "low": 0.5055555555555555, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7430303030303029, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4930555555555556, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4116970675635254, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5246927151765788, + "low": 0.5848668839583795, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.360821089959548, + "low": 1.0729441113071516, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 5.716666666666667, + "low": 4.227083333333334, + "mean": 4.927083333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 4.927083333333333, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 7.0, + "1073161778/train2": 3.0, + "1113371261/train0": 0.0, + "1113371261/train1": 4.0, + "1113371261/train2": 2.0, + "1151410346/train0": 7.0, + "1151410346/train1": 9.0, + "1151410346/train2": 10.0, + "1261930456/train0": 5.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 5.0, + "1324133122/train2": 7.0, + "1324642411/train0": 3.0, + "1324642411/train1": 9.0, + "1324642411/train2": 3.0, + "1375492786/train0": 15.0, + "1375492786/train1": 10.0, + "1375492786/train2": 15.0, + "141984615/train0": 1.0, + "141984615/train1": 6.0, + "141984615/train2": 4.0, + "1461069713/train0": 2.0, + "1461069713/train1": 0.0, + "1461069713/train2": 5.0, + "1480856810/train0": 2.0, + "1480856810/train1": 13.0, + "1480856810/train2": 1.0, + "1509980664/train0": 10.0, + "1509980664/train1": 4.0, + "1509980664/train2": 12.0, + "1522272686/train0": 8.0, + "1522272686/train1": 7.0, + "1522272686/train2": 8.0, + "1549324805/train0": 0.0, + "1549324805/train1": 2.0, + "1549324805/train2": 3.0, + "1578990937/train0": 15.0, + "1578990937/train1": 12.0, + "1578990937/train2": 0.0, + "1747178030/train0": 5.0, + "1747178030/train1": 9.0, + "1747178030/train2": 0.0, + "1790188787/train0": 9.0, + "1790188787/train1": 6.0, + "1790188787/train2": 0.0, + "1822210471/train0": 8.0, + "1822210471/train1": 5.0, + "1822210471/train2": 0.0, + "1845489013/train0": 5.0, + "1845489013/train1": 7.0, + "1845489013/train2": 8.0, + "1999022574/train0": 11.0, + "1999022574/train1": 9.0, + "1999022574/train2": 6.0, + "2012211287/train0": 0.0, + "2012211287/train1": 5.0, + "2012211287/train2": 1.0, + "2038594191/train0": 3.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 5.0, + "2061635888/train1": 6.0, + "2061635888/train2": 4.0, + "211245842/train0": 7.0, + "211245842/train1": 13.0, + "211245842/train2": 0.0, + "278008524/train0": 3.0, + "278008524/train1": 1.0, + "278008524/train2": 11.5, + "307068879/train0": 2.0, + "307068879/train1": 12.0, + "307068879/train2": 0.0, + "412102563/train0": 6.0, + "412102563/train1": 10.0, + "412102563/train2": 3.0, + "440577711/train0": 3.0, + "440577711/train1": 5.0, + "440577711/train2": 5.0, + "441046872/train0": 10.0, + "441046872/train1": 3.0, + "441046872/train2": 0.0, + "509430392/train0": 14.0, + "509430392/train1": 2.0, + "509430392/train2": 1.0, + "585904130/train0": 14.0, + "585904130/train1": 7.0, + "585904130/train2": 4.0, + "608016290/train0": 3.0, + "608016290/train1": 8.0, + "608016290/train2": 15.0, + "626486813/train0": 0.0, + "626486813/train1": 0.5, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 2.0, + "655884605/train2": 8.0, + "672328262/train0": 8.0, + "672328262/train1": 1.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 10.0, + "701667486/train2": 8.0, + "747260437/train0": 0.0, + "747260437/train1": 8.0, + "747260437/train2": 1.0, + "750448296/train0": 4.0, + "750448296/train1": 3.0, + "750448296/train2": 2.0, + "767747424/train0": 7.0, + "767747424/train1": 2.0, + "767747424/train2": 7.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 6.771717171717173, + "low": 4.363555555555555, + "mean": 5.475717171717173, + "n": 5, + "num_samples": 1000 + }, + "mean": 5.475717171717173, + "per_task": { + "LiftPegUpright-v1": 5.0, + "PickCube-v1": 7.666666666666667, + "PullCube-v1": 3.8533333333333335, + "PushCube-v1": 4.555555555555555, + "StackCube-v1": 6.303030303030303 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 5.409722222222222, + "low": 4.020833333333333, + "mean": 4.701388888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3875, + "low": 0.21875, + "mean": 0.3020833333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3020833333333333, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.29943434343434344, + "low": 0.10060606060606062, + "mean": 0.2174949494949495, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2174949494949495, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.30666666666666664, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2152777777777778, + "mean": 0.2847222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.041666666666666664, + "low": -0.05416666666666667, + "mean": -0.004166666666666667, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.004166666666666667, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.0, + "2038594191/train1": -1.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.06666666666666667, + "low": -0.02593939393939394, + "mean": 0.017616161616161613, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.017616161616161613, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.05333333333333334, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.034722222222222224, + "low": -0.05555555555555555, + "mean": -0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9239329974577736, + "low": 0.6214197847254884, + "mean": 0.7678461809070238, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7678461809070238, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 0.04538704454898834, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.061532557010650635, + "1324642411/train1": 0.1280023753643036, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.0042244791984558105, + "141984615/train1": 0.035802166908979416, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.009687979705631733, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.20681259036064148, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 2.0, + "1509980664/train1": 0.21804393827915192, + "1509980664/train2": 0.8189421892166138, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.02059849165380001, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.14965972304344177, + "1790188787/train1": 0.13741976022720337, + "1790188787/train2": 2.0, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 0.05387979745864868, + "2038594191/train1": 0.023384133353829384, + "2038594191/train2": 2.0, + "2054899126/train0": 1.052448581904173, + "2054899126/train1": 1.0664558708667755, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.11833827197551727, + "211245842/train1": 0.17156320810317993, + "211245842/train2": 0.17691531777381897, + "278008524/train0": 1.0485825426876545, + "278008524/train1": 0.1001356840133667, + "278008524/train2": 0.052897630259394646, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.12097799777984619, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.005386819248087704, + "626486813/train1": 0.00787139916792512, + "626486813/train2": 0.015375842805951834, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 0.06632735580205917, + "750448296/train0": 0.09317883538703124, + "750448296/train1": 0.11618362615505855, + "750448296/train2": 0.1037093202273051, + "767747424/train0": 0.5386653137393296, + "767747424/train1": 0.5581304961815476, + "767747424/train2": 0.5799880577251315 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8515048929266256, + "low": 0.6142033413514081, + "mean": 0.7215833358311028, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7215833358311028, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.666174189314867, + "PushCube-v1": 0.6184621560904715, + "StackCube-v1": 0.7894506924080126 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8554598261252977, + "low": 0.5858313379317729, + "mean": 0.7162999392102292, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.06609533377680539, + "low": -0.11685185874133215, + "mean": -0.02227188139077043, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.02227188139077043, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.9546129554510117, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -0.044937968254089355, + "1324642411/train1": 0.0215318500995636, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.09860976785421371, + "141984615/train1": -0.06703208014369011, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": -0.08421508502215147, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.11977851390838623, + "1480856810/train2": 0.0, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": -0.6153660863637924, + "1509980664/train2": -0.014467835426330566, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": 0.0, + "1549324805/train2": -0.08391634561121464, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.024892255663871765, + "1790188787/train1": 0.012652292847633362, + "1790188787/train2": 1.87523253262043, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.9461202025413513, + "2038594191/train1": -1.9766158666461706, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9475514180958271, + "2054899126/train1": -0.9335441291332245, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": -0.07257014513015747, + "211245842/train1": -0.019345209002494812, + "211245842/train2": -0.013993099331855774, + "278008524/train0": 1.0058475527912378, + "278008524/train1": 0.057400694116950035, + "278008524/train2": 0.010162640362977982, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.12097777426242828, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0002884765272028744, + "626486813/train1": 0.0027730564470402896, + "626486813/train2": 0.010277500085067004, + "636680137/train0": -0.027042903006076813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.9336726441979408, + "750448296/train0": -0.024414313957095146, + "750448296/train1": -0.0014095231890678406, + "750448296/train2": -0.013883829116821289, + "767747424/train0": -0.03453402826562524, + "767747424/train1": -0.015068845823407173, + "767747424/train2": 0.006788715720176697 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1356347863872846, + "low": -0.06278656556877787, + "mean": 0.026023538657087796, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.026023538657087796, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.10899627190859368, + "PushCube-v1": 0.226057977312141, + "StackCube-v1": 0.013055987881891655 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.05458070211009019, + "low": -0.12055642578343395, + "mean": -0.025519647232108075, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6801490995260085, + "low": 0.40305140065320294, + "mean": 0.5353515408005073, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5353515408005073, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.9546129554510117, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.08252748847007751, + "1324642411/train1": 1.8719976246356964, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.010953426361083984, + "141984615/train1": 0.001208961009979248, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.00689703319221735, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.7931874096393585, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 0.0, + "1509980664/train1": 1.781956061720848, + "1509980664/train2": 1.1810578107833862, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.09250334836542606, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.8503402769565582, + "1790188787/train1": 1.8625802397727966, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 1.9461202025413513, + "2038594191/train1": 1.9766158666461706, + "2038594191/train2": 0.0, + "2054899126/train0": 0.024834569543600082, + "2054899126/train1": 0.9335441291332245, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8816617280244827, + "211245842/train1": 0.04139396548271179, + "211245842/train2": 0.05696745216846466, + "278008524/train0": 0.02477378025650978, + "278008524/train1": 0.9375389106571674, + "278008524/train2": 1.0830512810498476, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 1.3000448346138, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8790220022201538, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.03930865635629743, + "626486813/train1": 0.027956475503742695, + "626486813/train2": 0.029523412231355906, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.6498691290616989, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.05877452343702316, + "750448296/train0": 0.6594814925144116, + "750448296/train1": 1.2543196479479473, + "750448296/train2": 1.246739347775777, + "767747424/train0": 0.5124941510148346, + "767747424/train1": 0.5076578482985497, + "767747424/train2": 0.48825800977647305 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8738215418463066, + "low": 0.17712627903981643, + "mean": 0.5098326246412377, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5098326246412377, + "per_task": { + "LiftPegUpright-v1": 0.5146939317385356, + "PickCube-v1": 0.0, + "PullCube-v1": 0.49621246912516653, + "PushCube-v1": 1.1673192588819399, + "StackCube-v1": 0.37093746346054657 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.6691991107056512, + "low": 0.41660342318290433, + "mean": 0.5429793549620727, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38333333333333336, + "low": 0.225, + "mean": 0.29930555555555555, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.29930555555555555, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4680000000000001, + "low": 0.12242424242424244, + "mean": 0.2953131313131313, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2953131313131313, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.25333333333333335, + "PushCube-v1": 0.6111111111111112, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2222222222222222, + "mean": 0.2986111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30416666666666664, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7191919191919192, + "low": 0.13511111111111113, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2638888888888889, + "low": 0.1388888888888889, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5434736852068454, + "low": 0.2965679521788843, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.330930626693398, + "low": 0.24626440485649642, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.485598717091812, + "low": 0.272009730946997, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.34731881885037813, + "low": 0.34731881885037813, + "mean": 0.34731881885037813, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.34731881885037813, + "per_seed": { + "1073161778/train0": 0.3473188188503786, + "1073161778/train1": 0.3473188188503786, + "1073161778/train2": 0.3473188188503786, + "1113371261/train0": 0.3473188188503786, + "1113371261/train1": 0.3473188188503786, + "1113371261/train2": 0.3473188188503786, + "1151410346/train0": 0.3473188188503786, + "1151410346/train1": 0.3473188188503786, + "1151410346/train2": 0.3473188188503786, + "1261930456/train0": 0.3473188188503786, + "1261930456/train1": 0.3473188188503786, + "1261930456/train2": 0.3473188188503786, + "1324133122/train0": 0.3473188188503786, + "1324133122/train1": 0.3473188188503786, + "1324133122/train2": 0.3473188188503786, + "1324642411/train0": 0.3473188188503786, + "1324642411/train1": 0.3473188188503786, + "1324642411/train2": 0.3473188188503786, + "1375492786/train0": 0.3473188188503786, + "1375492786/train1": 0.3473188188503786, + "1375492786/train2": 0.3473188188503786, + "141984615/train0": 0.3473188188503786, + "141984615/train1": 0.3473188188503786, + "141984615/train2": 0.3473188188503786, + "1461069713/train0": 0.3473188188503786, + "1461069713/train1": 0.3473188188503786, + "1461069713/train2": 0.3473188188503786, + "1480856810/train0": 0.3473188188503786, + "1480856810/train1": 0.3473188188503786, + "1480856810/train2": 0.3473188188503786, + "1509980664/train0": 0.3473188188503786, + "1509980664/train1": 0.3473188188503786, + "1509980664/train2": 0.3473188188503786, + "1522272686/train0": 0.3473188188503786, + "1522272686/train1": 0.3473188188503786, + "1522272686/train2": 0.3473188188503786, + "1549324805/train0": 0.3473188188503786, + "1549324805/train1": 0.3473188188503786, + "1549324805/train2": 0.3473188188503786, + "1578990937/train0": 0.3473188188503786, + "1578990937/train1": 0.3473188188503786, + "1578990937/train2": 0.3473188188503786, + "1747178030/train0": 0.3473188188503786, + "1747178030/train1": 0.3473188188503786, + "1747178030/train2": 0.3473188188503786, + "1790188787/train0": 0.3473188188503786, + "1790188787/train1": 0.3473188188503786, + "1790188787/train2": 0.3473188188503786, + "1822210471/train0": 0.3473188188503786, + "1822210471/train1": 0.3473188188503786, + "1822210471/train2": 0.3473188188503786, + "1845489013/train0": 0.3473188188503786, + "1845489013/train1": 0.3473188188503786, + "1845489013/train2": 0.3473188188503786, + "1999022574/train0": 0.3473188188503786, + "1999022574/train1": 0.3473188188503786, + "1999022574/train2": 0.3473188188503786, + "2012211287/train0": 0.3473188188503786, + "2012211287/train1": 0.3473188188503786, + "2012211287/train2": 0.3473188188503786, + "2038594191/train0": 0.3473188188503786, + "2038594191/train1": 0.3473188188503786, + "2038594191/train2": 0.3473188188503786, + "2054899126/train0": 0.3473188188503786, + "2054899126/train1": 0.3473188188503786, + "2054899126/train2": 0.3473188188503786, + "2061635888/train0": 0.3473188188503786, + "2061635888/train1": 0.3473188188503786, + "2061635888/train2": 0.3473188188503786, + "211245842/train0": 0.3473188188503786, + "211245842/train1": 0.3473188188503786, + "211245842/train2": 0.3473188188503786, + "278008524/train0": 0.3473188188503786, + "278008524/train1": 0.3473188188503786, + "278008524/train2": 0.3473188188503786, + "307068879/train0": 0.3473188188503786, + "307068879/train1": 0.3473188188503786, + "307068879/train2": 0.3473188188503786, + "412102563/train0": 0.3473188188503786, + "412102563/train1": 0.3473188188503786, + "412102563/train2": 0.3473188188503786, + "440577711/train0": 0.3473188188503786, + "440577711/train1": 0.3473188188503786, + "440577711/train2": 0.3473188188503786, + "441046872/train0": 0.3473188188503786, + "441046872/train1": 0.3473188188503786, + "441046872/train2": 0.3473188188503786, + "509430392/train0": 0.3473188188503786, + "509430392/train1": 0.3473188188503786, + "509430392/train2": 0.3473188188503786, + "585904130/train0": 0.3473188188503786, + "585904130/train1": 0.3473188188503786, + "585904130/train2": 0.3473188188503786, + "608016290/train0": 0.3473188188503786, + "608016290/train1": 0.3473188188503786, + "608016290/train2": 0.3473188188503786, + "626486813/train0": 0.3473188188503786, + "626486813/train1": 0.3473188188503786, + "626486813/train2": 0.3473188188503786, + "636680137/train0": 0.3473188188503786, + "636680137/train1": 0.3473188188503786, + "636680137/train2": 0.3473188188503786, + "655884605/train0": 0.3473188188503786, + "655884605/train1": 0.3473188188503786, + "655884605/train2": 0.3473188188503786, + "672328262/train0": 0.3473188188503786, + "672328262/train1": 0.3473188188503786, + "672328262/train2": 0.3473188188503786, + "701667486/train0": 0.3473188188503786, + "701667486/train1": 0.3473188188503786, + "701667486/train2": 0.3473188188503786, + "747260437/train0": 0.3473188188503786, + "747260437/train1": 0.3473188188503786, + "747260437/train2": 0.3473188188503786, + "750448296/train0": 0.3473188188503786, + "750448296/train1": 0.3473188188503786, + "750448296/train2": 0.3473188188503786, + "767747424/train0": 0.3473188188503786, + "767747424/train1": 0.3473188188503786, + "767747424/train2": 0.3473188188503786 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.34731881885037874, + "low": 0.34731881885037846, + "mean": 0.3473188188503786, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3473188188503786, + "per_task": { + "LiftPegUpright-v1": 0.34731881885037846, + "PickCube-v1": 0.3473188188503786, + "PullCube-v1": 0.34731881885037885, + "PushCube-v1": 0.3473188188503784, + "StackCube-v1": 0.3473188188503786 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.3473188188503777, + "low": 0.3473188188503777, + "mean": 0.3473188188503777, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "success_weighted_margin", + "tau": 0.3473188188503786 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..0bf7f4dae4c0ac2069e096d6ba0e5ffe11eb84c7 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28355827608242967, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3337636013451598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29522744379276233, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32662525863909614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.3473188188503786 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27481853311866083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.3473188188503786 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3131801253335776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.3473188188503786 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20532758727946646, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.3473188188503786 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21665335322366186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22061568738801626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.3473188188503786 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5566373654374286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5940825716503084, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5877928871421093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04538704454898834, + "selected_utility_gain_over_base": -1.9546129554510117, + "selector_regret": 1.9546129554510117, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1953526650841092, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.3473188188503786 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10311308876908776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.3473188188503786 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17894998490871183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.3473188188503786 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244761772076728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.061532557010650635, + "selected_utility_gain_over_base": -0.044937968254089355, + "selector_regret": 0.08252748847007751, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.3473188188503786 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48070996484280953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4884411626074505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16043123876023302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17011388186485987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15542103752775793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3988789611600871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0042244791984558105, + "selected_utility_gain_over_base": -0.09860976785421371, + "selector_regret": 0.010953426361083984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.3473188188503786 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37538333059509166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035802166908979416, + "selected_utility_gain_over_base": -0.06703208014369011, + "selector_regret": 0.001208961009979248, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.3473188188503786 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30448894670641713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.3473188188503786 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3050981300072402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.3473188188503786 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.7484074094930177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.3473188188503786 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.2910706981901193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.3473188188503786 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43921922377953376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5870819789979614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6097239173416052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5974323487206175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08863633481483887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.3473188188503786 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09154681464635614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.3473188188503786 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08595256684637552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.3473188188503786 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.4534903350362056, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.3473188188503786 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.1768234215359843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.3473188188503786 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5340001574841613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02059849165380001, + "selected_utility_gain_over_base": -0.08391634561121464, + "selector_regret": 0.09250334836542606, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.3473188188503786 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2255580985362039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21684013213485268, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2250806530775492, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3352371313866231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33370414261502274, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33681729037438607, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7256808008161852, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14965972304344177, + "selected_utility_gain_over_base": 0.024892255663871765, + "selector_regret": 1.8503402769565582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7407334225672018, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7188844098969164, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10685249697353241, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11932515699499706, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12054968766555163, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22822097403519162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23290921447763308, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23237107122905468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2830091920285253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2762933949412652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32995590919912554, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.08991720318616597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1018745030033766, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09743460718887578, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05387979745864868, + "selected_utility_gain_over_base": -1.9461202025413513, + "selector_regret": 1.9461202025413513, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39227093434151067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.023384133353829384, + "selected_utility_gain_over_base": -1.9766158666461706, + "selector_regret": 1.9766158666461706, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.48682363081170954, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.052448581904173, + "selected_utility_gain_over_base": -0.9475514180958271, + "selector_regret": 0.024834569543600082, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.3473188188503786 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4867067691213679, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0664558708667755, + "selected_utility_gain_over_base": -0.9335441291332245, + "selector_regret": 0.9335441291332245, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4238219648796263, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16138880459399857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.3473188188503786 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12788381852052721, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.3473188188503786 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13169501516378973, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.3473188188503786 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36446911237871227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11833827197551727, + "selected_utility_gain_over_base": -0.07257014513015747, + "selector_regret": 1.8816617280244827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.3473188188503786 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17691531777381897, + "selected_utility_gain_over_base": -0.013993099331855774, + "selector_regret": 0.05696745216846466, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.3473188188503786 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.4154810514015457, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0485825426876545, + "selected_utility_gain_over_base": 1.0058475527912378, + "selector_regret": 0.02477378025650978, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.3473188188503786 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.431062905506948, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1001356840133667, + "selected_utility_gain_over_base": 0.057400694116950035, + "selector_regret": 0.9375389106571674, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.3473188188503786 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.39125281310385696, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 11.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.052897630259394646, + "selected_utility_gain_over_base": 0.010162640362977982, + "selector_regret": 1.0830512810498476, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2797080480367002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.3473188188503786 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24778606153341362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.3473188188503786 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29487688188584205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.3473188188503786 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20585303592937382, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20493799872831076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.18125860724557305, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20172868695503152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24443771108596596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.3473188188503786 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19710526958727811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.3473188188503786 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.39582323274190295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3836105423742797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3982631871077599, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.3473188188503786 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26430287849596273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23332367249842212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2531734715082984, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18560038716365618, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2379598925580104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1808325968136315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.3473188188503786 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.41319658430741574, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.005386819248087704, + "selected_utility_gain_over_base": 0.0002884765272028744, + "selector_regret": 0.03930865635629743, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.3473188188503786 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.46817656253554407, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00787139916792512, + "selected_utility_gain_over_base": 0.0027730564470402896, + "selector_regret": 0.027956475503742695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.3473188188503786 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.3841254242297905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015375842805951834, + "selected_utility_gain_over_base": 0.010277500085067004, + "selector_regret": 0.029523412231355906, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.3473188188503786 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371194860392053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.3473188188503786 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2496823681236075, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.3473188188503786 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2912910141265051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.3473188188503786 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30368062376907057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.3473188188503786 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2713166038796752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.3473188188503786 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19620741382881715, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.3473188188503786 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042802515015988585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04466194101303914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.048270193237023314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3589247640152091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38798745623031156, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4021201653187239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.28297507531186483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.3473188188503786 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2525195047503382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.3473188188503786 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.5259115214300562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06632735580205917, + "selected_utility_gain_over_base": -1.9336726441979408, + "selector_regret": 0.05877452343702316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.3473188188503786 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.4190984327494606, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09317883538703124, + "selected_utility_gain_over_base": -0.024414313957095146, + "selector_regret": 0.6594814925144116, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.3473188188503786 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.33489946089892014, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11618362615505855, + "selected_utility_gain_over_base": -0.0014095231890678406, + "selector_regret": 1.2543196479479473, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.3473188188503786 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.31678157342373975, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1037093202273051, + "selected_utility_gain_over_base": -0.013883829116821289, + "selector_regret": 1.246739347775777, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.3473188188503786 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28067674068264487, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 7.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5386653137393296, + "selected_utility_gain_over_base": -0.03453402826562524, + "selector_regret": 0.5124941510148346, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.3473188188503786 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28220702596299274, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 2.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5581304961815476, + "selected_utility_gain_over_base": -0.015068845823407173, + "selector_regret": 0.5076578482985497, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.3473188188503786 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28169149441007413, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 7.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5799880577251315, + "selected_utility_gain_over_base": 0.006788715720176697, + "selector_regret": 0.48825800977647305, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.3473188188503786 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..a11dc53387354f6dc7e42837a1c83efaa31679e1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "predicted_margin": 0.21812074468121995, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 5.0, + "selected_success": 0.2, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.9498705387115478, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5146939317385356, + "success_selector_gap": 0.4, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.34731881885037846 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08871190543585684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 7.666666666666667, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.3473188188503786 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.38666666666666666, + "execute_generated": 0.38666666666666666, + "fallback_rate": 0.6133333333333333, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.33025922453409173, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 3.8533333333333335, + "selected_success": 0.30666666666666664, + "selected_success_gain_over_base": -0.05333333333333334, + "selected_utility": 0.666174189314867, + "selected_utility_gain_over_base": -0.10899627190859368, + "selector_regret": 0.49621246912516653, + "success_selector_gap": 0.25333333333333335, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.34731881885037885 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.7222222222222222, + "execute_generated": 0.7222222222222222, + "fallback_rate": 0.2777777777777778, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "predicted_margin": 0.4544794286452365, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 4.555555555555555, + "selected_success": 0.2777777777777778, + "selected_success_gain_over_base": 0.1111111111111111, + "selected_utility": 0.6184621560904715, + "selected_utility_gain_over_base": 0.226057977312141, + "selector_regret": 1.1673192588819399, + "success_selector_gap": 0.6111111111111112, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.3473188188503784 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.2727272727272727, + "execute_generated": 0.2727272727272727, + "fallback_rate": 0.7272727272727273, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "predicted_margin": 0.2862752255853337, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 6.303030303030303, + "selected_success": 0.30303030303030304, + "selected_success_gain_over_base": 0.030303030303030304, + "selected_utility": 0.7894506924080126, + "selected_utility_gain_over_base": 0.013055987881891655, + "selector_regret": 0.37093746346054657, + "success_selector_gap": 0.21212121212121213, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.3473188188503786 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..d1cc8ecaab77d65e0349b9ca781e7598c860b0b1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.347319` +Feature set: `chart_compat` +Target: `success_weighted_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4549 | 0.5451 | 0.2780 | 0.4621 | 0.5343 | 0.5343 | 0.1733 | 0.1264 | +| selection | 0.4903 | 0.5097 | 0.2774 | 0.4452 | 0.5419 | 0.5677 | 0.1806 | 0.1419 | +| eval | 0.3542 | 0.6458 | 0.2917 | 0.2847 | 0.5694 | 0.5486 | 0.2014 | 0.2986 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..2706c3e0547971eab692012771ffa1c44982d2e5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,20 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + }, + "source_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..a3d945b1021ea4fa647f2f4f5760bbc06c826384 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.3542 & 0.6458 & 0.2917 & 0.2847 & 0.5694 & 0.5486 & 0.2014 & 0.2986 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..601d7f3b02f91084eb580b305aaa600dddb73f9f --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0} +tau=0.347319 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..95217928ac19dcf6f4e5a62e3d84f6c6b944b51f --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test --feature-set chart_compat --selector-chart-feature-mode base_context_obs --target success_weighted_margin --model-types hgb_regressor,rf_regressor --k 16 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d350057766e40d2cafba288891d378a44fef56c8 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,15 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_compat +k: 16 +model_types: hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test +seed: 0 +selection_frac: 0.35 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +target: success_weighted_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c001552e076e38f762c1980a9ac2fdd590af6dc4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e988ba5908f4c1dcaea4a90cae956d5d9261d6db --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,7469 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.3402777777777778, + "fallback_rate": 0.6597222222222222, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.3263888888888889, + "selected_success_gain_over_base": 0.034722222222222224, + "selected_utility": 0.7918940216570465, + "selector_regret": 0.4818038984736227, + "success_selector_gap": 0.2638888888888889, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "chart_compat", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.33574007220216606, + "fallback_rate": 0.6642599277978339, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.5342960288808665, + "proposal_oracle_success": 0.5342960288808665, + "proposal_oracle_utility": 1.1336619752116286, + "selected_success": 0.4620938628158845, + "selected_success_gain_over_base": 0.18411552346570398, + "selected_utility": 1.0242384470577288, + "selector_regret": 0.2294462014204866, + "success_selector_gap": 0.1263537906137184, + "success_support_gap": 0.17328519855595667, + "support_gap": 0.3521794904517649 + }, + "k": 16, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.33548387096774196, + "fallback_rate": 0.6645161290322581, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.3741935483870968, + "selected_success_gain_over_base": 0.0967741935483871, + "selected_utility": 0.8367662633701618, + "selector_regret": 0.40473204348011005, + "success_selector_gap": 0.2064516129032258, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "model_types": [ + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "num_fit_candidates": 4175, + "num_fit_rows": 277, + "num_selection_candidates": 2335, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.10762542421499345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.03189693906883312, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06754539929905881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.26553206302513555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09430490853290777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7131073754447632, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02598150633275509, + "selected_utility_gain_over_base": -0.13570442236959934, + "selector_regret": 1.974018493667245, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.030876378763120833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": -0.14309335922015287, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.008253872743807733, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.224269539572444, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1048971638083458, + "selected_utility_gain_over_base": -1.8951028361916542, + "selector_regret": 0.049669139087200165, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37033219487548785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.03911909498003871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.030455682258546335, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.11082633957266808, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46540267632512955, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.0240145517043362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.8603807411681497, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0006744663114659488, + "selected_utility_gain_over_base": -1.999325533688534, + "selector_regret": 1.999325533688534, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.783211815160911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.45076360899263673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8057818243102283, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11457405239343643, + "selected_utility_gain_over_base": -0.02813490480184555, + "selector_regret": 0.10201253741979599, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.730922283654924, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.054690223187208176, + "selected_utility_gain_over_base": -0.028444822877645493, + "selector_regret": 0.03357108309864998, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.45361446292677865, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.03964445208865493, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4320928427625122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.641025431709982, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02024897001683712, + "selected_utility_gain_over_base": -0.07274852134287357, + "selector_regret": 0.021145423874258995, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3961628251752525, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 12, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.11147982914005515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3360583914662149, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.04057706719583071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3753131616482717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.0607210327125378, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7208522520964407, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5083763279916524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14406004548072815, + "selected_utility_gain_over_base": 0.03758952021598816, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.928122443422234, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06047999858856201, + "selected_utility_gain_over_base": -0.064287468791008, + "selector_regret": 1.939520001411438, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09105107967198207, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.425471802912922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.23502340911767072, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.01765980271334038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5653215567693661, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11142203211784363, + "selected_utility_gain_over_base": -0.051584646105766296, + "selector_regret": 0.025906652212142944, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14846214977339026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5976092755124571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6258640055379121, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1094239205121994, + "selected_utility_gain_over_base": -0.08148449659347534, + "selector_regret": 1.8905760794878006, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.042155400349067146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07276587335777485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.5199957201646186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2296842211404253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23966364189076328, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.10762542421499345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.024889983439393283, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06754539929905881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.26553206302513555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09430490853290777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.359836178887591, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.05928441239142939, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.34842505669703017, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0079664591467008, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.1026011613012325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13291174173355103, + "selected_utility_gain_over_base": -1.867088258266449, + "selector_regret": 1.867088258266449, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09793626053678911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": -0.20923725476749722, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.030455682258546335, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03946288302540779, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46540267632512955, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.0240145517043362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05349260475486517, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 1.4944343850469877, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.783211815160911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.6018704158695247, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01966226100921631, + "selected_utility_gain_over_base": -0.08317198604345322, + "selector_regret": 0.017348866909742355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6786514544067791, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10845309495925903, + "selected_utility_gain_over_base": -0.03425586223602295, + "selector_regret": 1.891546905040741, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.5715662758164363, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01698315143585205, + "selected_utility_gain_over_base": -0.06615189462900162, + "selector_regret": 0.050428032875061035, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.4303226851491588, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.019438254631384468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4320928427625122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.06874312683552916, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018512330949306488, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.42208074329558476, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.15923551786737147, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3360583914662149, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.04057706719583071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3462447613139206, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.0607210327125378, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8035469324826213, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5702108833985109, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13300009071826935, + "selected_utility_gain_over_base": 0.026529565453529358, + "selector_regret": 1.8669999092817307, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.9487161705951446, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1179583821973945, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.4513022033896152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.32219394509834176, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.558522723271604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13658663630485535, + "selected_utility_gain_over_base": -0.026420041918754578, + "selector_regret": 0.6762891709804535, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.03128933348441161, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14846214977339026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5976092755124571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8125686645507812, + "selected_utility_gain_over_base": -0.020841360092163086, + "selector_regret": 1.1874313354492188, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5604320834866071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18476036190986633, + "selected_utility_gain_over_base": -0.006148055195808411, + "selector_regret": 0.02819681167602539, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.042155400349067146, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07276587335777485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.5199957201646186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.25659152366583776, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2690908362458581, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.10762542421499345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.045483710612303836, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06754539929905881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.26553206302513555, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09430490853290777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7131073754447632, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18884079158306122, + "selected_utility_gain_over_base": 0.027154862880706787, + "selector_regret": 1.8111592084169388, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.030876378763120833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.3278313295241196, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036922433762811124, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8030206489107127, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0775822177529335, + "selected_utility_gain_over_base": -1.9224177822470665, + "selector_regret": 1.9224177822470665, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37033219487548785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": -0.20923725476749722, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.030455682258546335, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46540267632512955, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08007363229990005, + "selected_utility_gain_over_base": -1.9199263677001, + "selector_regret": 1.9199263677001, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.0240145517043362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.8486151157121384, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0019492110004648566, + "selected_utility_gain_over_base": -1.9980507889995351, + "selector_regret": 1.9980507889995351, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.783211815160911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.45076360899263673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6361412904352629, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.857291042804718, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.730922283654924, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02323226071894169, + "selected_utility_gain_over_base": -0.05990278534591198, + "selector_regret": 0.05241936258971691, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.45361446292677865, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.019438254631384468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4320928427625122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.06874312683552916, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37136561643642657, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.15923551786737147, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3360583914662149, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.04057706719583071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.008587002754211426, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3753131616482717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.0607210327125378, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7208522520964407, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.49404811032777285, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11692000925540924, + "selected_utility_gain_over_base": 0.01044948399066925, + "selector_regret": 1.8830799907445908, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.9487161705951446, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10894416272640228, + "selected_utility_gain_over_base": -0.015823304653167725, + "selector_regret": 1.8910558372735977, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09592156335756839, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.425471802912922, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.26627686358563757, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.558522723271604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19850653409957886, + "selected_utility_gain_over_base": 0.03549985587596893, + "selector_regret": 0.01596677303314209, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14846214977339026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5976092755124571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18445390462875366, + "selected_utility_gain_over_base": -0.6489561200141907, + "selector_regret": 1.8155460953712463, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6258640055379121, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12799203395843506, + "selected_utility_gain_over_base": -0.06291638314723969, + "selector_regret": 0.10589073598384857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.042155400349067146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07276587335777485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.5199957201646186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2296842211404253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2690908362458581, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.461885278016425, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 40, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_regressor", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.33548387096774196, + "fallback_rate": 0.6645161290322581, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.3741935483870968, + "selected_success_gain_over_base": 0.0967741935483871, + "selected_utility": 0.8367662633701618, + "selector_regret": 0.40473204348011005, + "success_selector_gap": 0.2064516129032258, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.40555555555555556, + "low": 0.2520833333333333, + "mean": 0.3256944444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3256944444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 1.0, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.48484848484848486, + "low": 0.06666666666666667, + "mean": 0.27272727272727276, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.27272727272727276, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.36363636363636365 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.4166666666666667, + "low": 0.2708333333333333, + "mean": 0.3402777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.40555555555555556, + "low": 0.2520833333333333, + "mean": 0.3256944444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3256944444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 1.0, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.48484848484848486, + "low": 0.06666666666666667, + "mean": 0.27272727272727276, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.27272727272727276, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.36363636363636365 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.4166666666666667, + "low": 0.2708333333333333, + "mean": 0.3402777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7479166666666667, + "low": 0.5944444444444446, + "mean": 0.6743055555555555, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6743055555555555, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.75, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9333333333333332, + "low": 0.5151515151515151, + "mean": 0.7272727272727273, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7272727272727273, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.6666666666666666, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.7291666666666666, + "low": 0.5833333333333334, + "mean": 0.6597222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4430555555555556, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7206060606060607, + "low": 0.2206060606060606, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4416681086289573, + "low": 0.28068214256651913, + "mean": 0.36243930779238387, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.36243930779238387, + "per_seed": { + "1073161778/train0": 0.06754539929905881, + "1073161778/train1": 0.06754539929905881, + "1073161778/train2": 0.06754539929905881, + "1113371261/train0": 0.01069560631150106, + "1113371261/train1": 0.03128933348441161, + "1113371261/train2": 0.01069560631150106, + "1151410346/train0": 0.042155400349067146, + "1151410346/train1": 0.042155400349067146, + "1151410346/train2": 0.042155400349067146, + "1261930456/train0": 0.09105107967198207, + "1261930456/train1": 0.1179583821973945, + "1261930456/train2": 0.09592156335756839, + "1324133122/train0": 0.07276587335777485, + "1324133122/train1": 0.07276587335777485, + "1324133122/train2": 0.07276587335777485, + "1324642411/train0": 0.5083763279916524, + "1324642411/train1": 0.5702108833985109, + "1324642411/train2": 0.49404811032777285, + "1375492786/train0": -0.03964445208865493, + "1375492786/train1": 0.019438254631384468, + "1375492786/train2": 0.019438254631384468, + "141984615/train0": 0.45076360899263673, + "141984615/train1": 0.6018704158695247, + "141984615/train2": 0.45076360899263673, + "1461069713/train0": 0.030876378763120833, + "1461069713/train1": 0.05928441239142939, + "1461069713/train2": 0.030876378763120833, + "1480856810/train0": 0.23502340911767072, + "1480856810/train1": 0.32219394509834176, + "1480856810/train2": 0.26627686358563757, + "1509980664/train0": 0.5976092755124571, + "1509980664/train1": 0.5976092755124571, + "1509980664/train2": 0.5976092755124571, + "1522272686/train0": -0.07343996006022091, + "1522272686/train1": -0.07343996006022091, + "1522272686/train2": -0.07343996006022091, + "1549324805/train0": 0.04057706719583071, + "1549324805/train1": 0.04057706719583071, + "1549324805/train2": 0.04057706719583071, + "1578990937/train0": 0.14846214977339026, + "1578990937/train1": 0.14846214977339026, + "1578990937/train2": 0.14846214977339026, + "1747178030/train0": -0.09430490853290777, + "1747178030/train1": -0.09430490853290777, + "1747178030/train2": -0.09430490853290777, + "1790188787/train0": 1.928122443422234, + "1790188787/train1": 1.9487161705951446, + "1790188787/train2": 1.9487161705951446, + "1822210471/train0": 0.3753131616482717, + "1822210471/train1": 0.3462447613139206, + "1822210471/train2": 0.3753131616482717, + "1845489013/train0": 0.46540267632512955, + "1845489013/train1": 0.46540267632512955, + "1845489013/train2": 0.46540267632512955, + "1999022574/train0": 0.23966364189076328, + "1999022574/train1": 0.2690908362458581, + "1999022574/train2": 0.2690908362458581, + "2012211287/train0": -0.10762542421499345, + "2012211287/train1": -0.10762542421499345, + "2012211287/train2": -0.10762542421499345, + "2038594191/train0": 0.4320928427625122, + "2038594191/train1": 0.4320928427625122, + "2038594191/train2": 0.4320928427625122, + "2054899126/train0": 0.8102161823738483, + "2054899126/train1": 0.7623409522984086, + "2054899126/train2": 0.5871931326735697, + "2061635888/train0": -0.01765980271334038, + "2061635888/train1": 0.01069560631150106, + "2061635888/train2": 0.01069560631150106, + "211245842/train0": 0.6258640055379121, + "211245842/train1": 0.5604320834866071, + "211245842/train2": 0.6258640055379121, + "278008524/train0": 0.18325703686238062, + "278008524/train1": 0.18325703686238062, + "278008524/train2": 0.18325703686238062, + "307068879/train0": 0.03189693906883312, + "307068879/train1": 0.024889983439393283, + "307068879/train2": 0.045483710612303836, + "412102563/train0": 0.2296842211404253, + "412102563/train1": 0.25659152366583776, + "412102563/train2": 0.2296842211404253, + "440577711/train0": 0.7208522520964407, + "440577711/train1": 0.8035469324826213, + "440577711/train2": 0.7208522520964407, + "441046872/train0": 0.01069560631150106, + "441046872/train1": 0.01069560631150106, + "441046872/train2": 0.01069560631150106, + "509430392/train0": 1.425471802912922, + "509430392/train1": 1.4513022033896152, + "509430392/train2": 1.425471802912922, + "585904130/train0": 0.783211815160911, + "585904130/train1": 0.783211815160911, + "585904130/train2": 0.783211815160911, + "608016290/train0": -0.26553206302513555, + "608016290/train1": -0.26553206302513555, + "608016290/train2": -0.26553206302513555, + "626486813/train0": -0.05953940375790834, + "626486813/train1": 0.18621980420068318, + "626486813/train2": 0.1759229406142279, + "636680137/train0": 0.730922283654924, + "636680137/train1": 0.5715662758164363, + "636680137/train2": 0.730922283654924, + "655884605/train0": 0.5653215567693661, + "655884605/train1": 0.558522723271604, + "655884605/train2": 0.558522723271604, + "672328262/train0": 0.37033219487548785, + "672328262/train1": -0.09793626053678911, + "672328262/train2": 0.37033219487548785, + "701667486/train0": 1.5199957201646186, + "701667486/train1": 1.5199957201646186, + "701667486/train2": 1.5199957201646186, + "747260437/train0": 0.11147982914005515, + "747260437/train1": 0.15923551786737147, + "747260437/train2": 0.15923551786737147, + "750448296/train0": 0.8358427629109161, + "750448296/train1": 0.6027052046516154, + "750448296/train2": 0.5885351499944433, + "767747424/train0": 0.5165554186299326, + "767747424/train1": 0.51883899857906, + "767747424/train2": 0.45152492482904577 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5687326821389177, + "low": 0.022198286261117795, + "mean": 0.29292931684875084, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.29292931684875084, + "per_task": { + "LiftPegUpright-v1": -0.052165254117878486, + "PickCube-v1": -0.07343996006022091, + "PullCube-v1": 0.3834765656041302, + "PushCube-v1": 0.8374506997361121, + "StackCube-v1": 0.3693245330816113 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.45751546084741257, + "low": 0.31059806076575375, + "mean": 0.3820817077451673, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6722222222222223, + "low": 0.5055555555555555, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7430303030303029, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4930555555555556, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4116970675635254, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5246927151765788, + "low": 0.5848668839583795, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.360821089959548, + "low": 1.0729441113071516, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 4.679166666666666, + "low": 3.3569444444444447, + "mean": 4.027083333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 4.027083333333334, + "per_seed": { + "1073161778/train0": 9.0, + "1073161778/train1": 1.0, + "1073161778/train2": 4.0, + "1113371261/train0": 0.0, + "1113371261/train1": 5.0, + "1113371261/train2": 2.0, + "1151410346/train0": 7.0, + "1151410346/train1": 9.0, + "1151410346/train2": 2.0, + "1261930456/train0": 0.0, + "1261930456/train1": 10.0, + "1261930456/train2": 4.0, + "1324133122/train0": 0.0, + "1324133122/train1": 2.0, + "1324133122/train2": 1.0, + "1324642411/train0": 12.0, + "1324642411/train1": 2.0, + "1324642411/train2": 0.0, + "1375492786/train0": 10.0, + "1375492786/train1": 1.0, + "1375492786/train2": 2.0, + "141984615/train0": 10.0, + "141984615/train1": 1.0, + "141984615/train2": 11.0, + "1461069713/train0": 2.0, + "1461069713/train1": 0.0, + "1461069713/train2": 5.0, + "1480856810/train0": 2.0, + "1480856810/train1": 13.0, + "1480856810/train2": 1.0, + "1509980664/train0": 10.0, + "1509980664/train1": 9.0, + "1509980664/train2": 4.0, + "1522272686/train0": 7.0, + "1522272686/train1": 7.0, + "1522272686/train2": 6.0, + "1549324805/train0": 6.0, + "1549324805/train1": 0.0, + "1549324805/train2": 12.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 3.0, + "1747178030/train1": 1.0, + "1747178030/train2": 9.0, + "1790188787/train0": 4.0, + "1790188787/train1": 3.0, + "1790188787/train2": 10.0, + "1822210471/train0": 7.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 2.0, + "1845489013/train2": 10.0, + "1999022574/train0": 5.0, + "1999022574/train1": 8.0, + "1999022574/train2": 7.0, + "2012211287/train0": 3.0, + "2012211287/train1": 7.0, + "2012211287/train2": 1.0, + "2038594191/train0": 3.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 6.0, + "2054899126/train1": 0.0, + "2054899126/train2": 2.5, + "2061635888/train0": 2.0, + "2061635888/train1": 0.0, + "2061635888/train2": 2.0, + "211245842/train0": 4.0, + "211245842/train1": 2.0, + "211245842/train2": 10.0, + "278008524/train0": 0.0, + "278008524/train1": 4.5, + "278008524/train2": 0.5, + "307068879/train0": 4.0, + "307068879/train1": 0.0, + "307068879/train2": 4.0, + "412102563/train0": 2.0, + "412102563/train1": 10.0, + "412102563/train2": 5.0, + "440577711/train0": 0.0, + "440577711/train1": 3.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 0.0, + "509430392/train0": 14.0, + "509430392/train1": 2.0, + "509430392/train2": 1.0, + "585904130/train0": 4.0, + "585904130/train1": 15.0, + "585904130/train2": 1.0, + "608016290/train0": 2.0, + "608016290/train1": 1.0, + "608016290/train2": 14.0, + "626486813/train0": 7.0, + "626486813/train1": 7.5, + "626486813/train2": 3.0, + "636680137/train0": 1.0, + "636680137/train1": 2.0, + "636680137/train2": 0.0, + "655884605/train0": 5.0, + "655884605/train1": 2.0, + "655884605/train2": 3.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 2.0, + "701667486/train0": 1.0, + "701667486/train1": 6.0, + "701667486/train2": 1.0, + "747260437/train0": 15.0, + "747260437/train1": 0.0, + "747260437/train2": 4.0, + "750448296/train0": 2.6666666666666665, + "750448296/train1": 1.6666666666666667, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 11.25, + "767747424/train1": 8.75, + "767747424/train2": 3.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 5.607757575757576, + "low": 3.766868686868687, + "mean": 4.545535353535353, + "n": 5, + "num_samples": 1000 + }, + "mean": 4.545535353535353, + "per_task": { + "LiftPegUpright-v1": 3.466666666666667, + "PickCube-v1": 6.666666666666667, + "PullCube-v1": 4.16, + "PushCube-v1": 4.555555555555555, + "StackCube-v1": 3.878787878787879 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 4.833333333333333, + "low": 3.4444444444444446, + "mean": 4.125, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.43402777777777773, + "low": 0.2638888888888889, + "mean": 0.3506944444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3506944444444445, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3857777777777778, + "low": 0.10933333333333332, + "mean": 0.2588282828282828, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2588282828282828, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.4444444444444444, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.4027777777777778, + "low": 0.2569444444444444, + "mean": 0.3263888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.09444444444444443, + "low": -0.0013888888888888913, + "mean": 0.04444444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.04444444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": -1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": -0.25, + "767747424/train1": 0.0, + "767747424/train2": -0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.17006060606060608, + "low": -0.005333333333333334, + "mean": 0.05894949494949495, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.05894949494949495, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.013333333333333334, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.09027777777777778, + "low": -0.020833333333333332, + "mean": 0.034722222222222224, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0146110169916938, + "low": 0.7144105405048322, + "mean": 0.8586056652164795, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.8586056652164795, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 0.13300009071826935, + "1324642411/train2": 0.11692000925540924, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.01966226100921631, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 2.0, + "1509980664/train1": 0.8125686645507812, + "1509980664/train2": 0.18445390462875366, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.06047999858856201, + "1790188787/train1": 2.0, + "1790188787/train2": 0.10894416272640228, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 0.08007363229990005, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.052448581904173, + "2054899126/train1": 1.0664558708667755, + "2054899126/train2": 1.0387911088764668, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.1094239205121994, + "211245842/train1": 0.18476036190986633, + "211245842/train2": 0.12799203395843506, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.12097799777984619, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.09919407218694687, + "585904130/train1": 0.04792279005050659, + "585904130/train2": 2.0, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.054690223187208176, + "636680137/train1": 0.01698315143585205, + "636680137/train2": 0.02323226071894169, + "655884605/train0": 0.11142203211784363, + "655884605/train1": 0.13658663630485535, + "655884605/train2": 0.19850653409957886, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7116076741367579, + "750448296/train1": 0.7338168621063232, + "750448296/train2": 1.3643324971199036, + "767747424/train0": 0.039441852990421467, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.08047536047524773 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9119433828285246, + "low": 0.6748840806554072, + "mean": 0.7992690209528817, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7992690209528817, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7355007326719352, + "PushCube-v1": 0.9414158223403825, + "StackCube-v1": 0.7855989084099279 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.933828907855463, + "low": 0.6560619380147222, + "mean": 0.7918940216570465, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.16107782928621522, + "low": -0.01788619253784418, + "mean": 0.0684876029186853, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.0684876029186853, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.03758952021598816, + "1324642411/train1": 0.026529565453529358, + "1324642411/train2": 0.01044948399066925, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": -0.08317198604345322, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": -0.020841360092163086, + "1509980664/train2": -0.6489561200141907, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": -0.064287468791008, + "1790188787/train1": 1.87523253262043, + "1790188787/train2": -0.015823304653167725, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": -1.9199263677001, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9475514180958271, + "2054899126/train1": -0.9335441291332245, + "2054899126/train2": -0.9612088911235332, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": -0.08148449659347534, + "211245842/train1": -0.006148055195808411, + "211245842/train2": -0.06291638314723969, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.12097777426242828, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": -0.0825214758515358, + "585904130/train1": -0.13379275798797607, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": -0.028444822877645493, + "636680137/train1": -0.06615189462900162, + "636680137/train2": -0.05990278534591198, + "655884605/train0": -0.051584646105766296, + "655884605/train1": -0.026420041918754578, + "655884605/train2": 0.03549985587596893, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.5940145247926315, + "750448296/train1": 0.6162237127621969, + "750448296/train2": 1.246739347775777, + "767747424/train0": -0.5337574890145333, + "767747424/train1": 0.0, + "767747424/train2": -0.4927239815297071 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3294069861372312, + "low": -0.021960996354154, + "mean": 0.10370922377886664, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.10370922377886664, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.039669728551525625, + "PushCube-v1": 0.549011643562052, + "StackCube-v1": 0.009204203883806864 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.15569836555773187, + "low": -0.05393566925866051, + "mean": 0.050074435214709306, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5985275688894842, + "low": 0.3415398011583925, + "mean": 0.4616630104891455, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4616630104891455, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.0, + "1324642411/train1": 1.8669999092817307, + "1324642411/train2": 1.8830799907445908, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.017348866909742355, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.9129659235477448, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 0.0, + "1509980664/train1": 1.1874313354492188, + "1509980664/train2": 1.8155460953712463, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.008587002754211426, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.939520001411438, + "1790188787/train1": 0.0, + "1790188787/train2": 1.8910558372735977, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 1.9199263677001, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.024834569543600082, + "2054899126/train1": 0.9335441291332245, + "2054899126/train2": 0.9612088911235332, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8905760794878006, + "211245842/train1": 0.02819681167602539, + "211245842/train2": 0.10589073598384857, + "278008524/train0": 1.0306213330477476, + "278008524/train1": 0.9949396047741175, + "278008524/train2": 1.0932139214128256, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.3000448346138, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8790220022201538, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.9008059278130531, + "585904130/train1": 1.9520772099494934, + "585904130/train2": 0.0, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.03959713288350031, + "626486813/train1": 0.030729531950782984, + "626486813/train2": 0.03980091231642291, + "636680137/train0": 0.03357108309864998, + "636680137/train1": 0.050428032875061035, + "636680137/train2": 0.05241936258971691, + "655884605/train0": 0.025906652212142944, + "655884605/train1": 0.6762891709804535, + "655884605/train2": 0.01596677303314209, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.041052653764685, + "750448296/train1": 0.6366864119966825, + "750448296/train2": 0.0, + "767747424/train0": 1.0117176117637428, + "767747424/train1": 0.4925890024751425, + "767747424/train2": 0.9877707070263568 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6724039930375435, + "low": 0.1777410116340175, + "mean": 0.43759654233243905, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.43759654233243905, + "per_task": { + "LiftPegUpright-v1": 0.5146939317385356, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4549120608600788, + "PushCube-v1": 0.8443655926320288, + "StackCube-v1": 0.3740111264315518 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.6083351303863714, + "low": 0.3587040142305745, + "mean": 0.4818038984736227, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.33819444444444446, + "low": 0.19166666666666668, + "mean": 0.25902777777777775, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.25902777777777775, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.0, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.38311111111111107, + "low": 0.12242424242424244, + "mean": 0.2566464646464647, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2566464646464647, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.22666666666666666, + "PushCube-v1": 0.4444444444444444, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3402777777777778, + "low": 0.19444444444444445, + "mean": 0.2638888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30416666666666664, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7191919191919192, + "low": 0.13511111111111113, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2638888888888889, + "low": 0.1388888888888889, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5434736852068454, + "low": 0.2965679521788843, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.330930626693398, + "low": 0.24626440485649642, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.485598717091812, + "low": 0.272009730946997, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4618852780164247, + "low": 0.4618852780164247, + "mean": 0.4618852780164247, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4618852780164247, + "per_seed": { + "1073161778/train0": 0.461885278016425, + "1073161778/train1": 0.461885278016425, + "1073161778/train2": 0.461885278016425, + "1113371261/train0": 0.461885278016425, + "1113371261/train1": 0.461885278016425, + "1113371261/train2": 0.461885278016425, + "1151410346/train0": 0.461885278016425, + "1151410346/train1": 0.461885278016425, + "1151410346/train2": 0.461885278016425, + "1261930456/train0": 0.461885278016425, + "1261930456/train1": 0.461885278016425, + "1261930456/train2": 0.461885278016425, + "1324133122/train0": 0.461885278016425, + "1324133122/train1": 0.461885278016425, + "1324133122/train2": 0.461885278016425, + "1324642411/train0": 0.461885278016425, + "1324642411/train1": 0.461885278016425, + "1324642411/train2": 0.461885278016425, + "1375492786/train0": 0.461885278016425, + "1375492786/train1": 0.461885278016425, + "1375492786/train2": 0.461885278016425, + "141984615/train0": 0.461885278016425, + "141984615/train1": 0.461885278016425, + "141984615/train2": 0.461885278016425, + "1461069713/train0": 0.461885278016425, + "1461069713/train1": 0.461885278016425, + "1461069713/train2": 0.461885278016425, + "1480856810/train0": 0.461885278016425, + "1480856810/train1": 0.461885278016425, + "1480856810/train2": 0.461885278016425, + "1509980664/train0": 0.461885278016425, + "1509980664/train1": 0.461885278016425, + "1509980664/train2": 0.461885278016425, + "1522272686/train0": 0.461885278016425, + "1522272686/train1": 0.461885278016425, + "1522272686/train2": 0.461885278016425, + "1549324805/train0": 0.461885278016425, + "1549324805/train1": 0.461885278016425, + "1549324805/train2": 0.461885278016425, + "1578990937/train0": 0.461885278016425, + "1578990937/train1": 0.461885278016425, + "1578990937/train2": 0.461885278016425, + "1747178030/train0": 0.461885278016425, + "1747178030/train1": 0.461885278016425, + "1747178030/train2": 0.461885278016425, + "1790188787/train0": 0.461885278016425, + "1790188787/train1": 0.461885278016425, + "1790188787/train2": 0.461885278016425, + "1822210471/train0": 0.461885278016425, + "1822210471/train1": 0.461885278016425, + "1822210471/train2": 0.461885278016425, + "1845489013/train0": 0.461885278016425, + "1845489013/train1": 0.461885278016425, + "1845489013/train2": 0.461885278016425, + "1999022574/train0": 0.461885278016425, + "1999022574/train1": 0.461885278016425, + "1999022574/train2": 0.461885278016425, + "2012211287/train0": 0.461885278016425, + "2012211287/train1": 0.461885278016425, + "2012211287/train2": 0.461885278016425, + "2038594191/train0": 0.461885278016425, + "2038594191/train1": 0.461885278016425, + "2038594191/train2": 0.461885278016425, + "2054899126/train0": 0.461885278016425, + "2054899126/train1": 0.461885278016425, + "2054899126/train2": 0.461885278016425, + "2061635888/train0": 0.461885278016425, + "2061635888/train1": 0.461885278016425, + "2061635888/train2": 0.461885278016425, + "211245842/train0": 0.461885278016425, + "211245842/train1": 0.461885278016425, + "211245842/train2": 0.461885278016425, + "278008524/train0": 0.461885278016425, + "278008524/train1": 0.461885278016425, + "278008524/train2": 0.461885278016425, + "307068879/train0": 0.461885278016425, + "307068879/train1": 0.461885278016425, + "307068879/train2": 0.461885278016425, + "412102563/train0": 0.461885278016425, + "412102563/train1": 0.461885278016425, + "412102563/train2": 0.461885278016425, + "440577711/train0": 0.461885278016425, + "440577711/train1": 0.461885278016425, + "440577711/train2": 0.461885278016425, + "441046872/train0": 0.461885278016425, + "441046872/train1": 0.461885278016425, + "441046872/train2": 0.461885278016425, + "509430392/train0": 0.461885278016425, + "509430392/train1": 0.461885278016425, + "509430392/train2": 0.461885278016425, + "585904130/train0": 0.461885278016425, + "585904130/train1": 0.461885278016425, + "585904130/train2": 0.461885278016425, + "608016290/train0": 0.461885278016425, + "608016290/train1": 0.461885278016425, + "608016290/train2": 0.461885278016425, + "626486813/train0": 0.461885278016425, + "626486813/train1": 0.461885278016425, + "626486813/train2": 0.461885278016425, + "636680137/train0": 0.461885278016425, + "636680137/train1": 0.461885278016425, + "636680137/train2": 0.461885278016425, + "655884605/train0": 0.461885278016425, + "655884605/train1": 0.461885278016425, + "655884605/train2": 0.461885278016425, + "672328262/train0": 0.461885278016425, + "672328262/train1": 0.461885278016425, + "672328262/train2": 0.461885278016425, + "701667486/train0": 0.461885278016425, + "701667486/train1": 0.461885278016425, + "701667486/train2": 0.461885278016425, + "747260437/train0": 0.461885278016425, + "747260437/train1": 0.461885278016425, + "747260437/train2": 0.461885278016425, + "750448296/train0": 0.4618852780164249, + "750448296/train1": 0.4618852780164249, + "750448296/train2": 0.4618852780164249, + "767747424/train0": 0.461885278016425, + "767747424/train1": 0.461885278016425, + "767747424/train2": 0.461885278016425 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4618852780164249, + "low": 0.46188527801642476, + "mean": 0.4618852780164248, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4618852780164248, + "per_task": { + "LiftPegUpright-v1": 0.46188527801642487, + "PickCube-v1": 0.4618852780164249, + "PullCube-v1": 0.46188527801642476, + "PushCube-v1": 0.46188527801642487, + "StackCube-v1": 0.46188527801642476 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.46188527801642476, + "low": 0.46188527801642476, + "mean": 0.46188527801642476, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "success_weighted_margin", + "tau": 0.461885278016425 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..d3a553bd8dce781c72f70615541343df9477c892 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06754539929905881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06754539929905881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06754539929905881, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.461885278016425 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.03128933348441161, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.461885278016425 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.461885278016425 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.042155400349067146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.461885278016425 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.042155400349067146, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.042155400349067146, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.461885278016425 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09105107967198207, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1179583821973945, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09592156335756839, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07276587335777485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.461885278016425 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07276587335777485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.461885278016425 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07276587335777485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.461885278016425 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5083763279916524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14406004548072815, + "selected_utility_gain_over_base": 0.03758952021598816, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.461885278016425 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5702108833985109, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13300009071826935, + "selected_utility_gain_over_base": 0.026529565453529358, + "selector_regret": 1.8669999092817307, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.49404811032777285, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11692000925540924, + "selected_utility_gain_over_base": 0.01044948399066925, + "selector_regret": 1.8830799907445908, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.03964445208865493, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.019438254631384468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.019438254631384468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.45076360899263673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.461885278016425 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.6018704158695247, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01966226100921631, + "selected_utility_gain_over_base": -0.08317198604345322, + "selector_regret": 0.017348866909742355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.461885278016425 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.45076360899263673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.461885278016425 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.030876378763120833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.461885278016425 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.05928441239142939, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.461885278016425 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.030876378763120833, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.461885278016425 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.23502340911767072, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.32219394509834176, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.26627686358563757, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5976092755124571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5976092755124571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8125686645507812, + "selected_utility_gain_over_base": -0.020841360092163086, + "selector_regret": 1.1874313354492188, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5976092755124571, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18445390462875366, + "selected_utility_gain_over_base": -0.6489561200141907, + "selector_regret": 1.8155460953712463, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.461885278016425 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.461885278016425 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.461885278016425 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.04057706719583071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.461885278016425 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.04057706719583071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.461885278016425 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.04057706719583071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.008587002754211426, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.461885278016425 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14846214977339026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14846214977339026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14846214977339026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09430490853290777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09430490853290777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09430490853290777, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.928122443422234, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06047999858856201, + "selected_utility_gain_over_base": -0.064287468791008, + "selector_regret": 1.939520001411438, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.9487161705951446, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.9487161705951446, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10894416272640228, + "selected_utility_gain_over_base": -0.015823304653167725, + "selector_regret": 1.8910558372735977, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3753131616482717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3462447613139206, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3753131616482717, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46540267632512955, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46540267632512955, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46540267632512955, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08007363229990005, + "selected_utility_gain_over_base": -1.9199263677001, + "selector_regret": 1.9199263677001, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23966364189076328, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2690908362458581, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2690908362458581, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.10762542421499345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.10762542421499345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.10762542421499345, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4320928427625122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4320928427625122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4320928427625122, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8102161823738483, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 6.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.052448581904173, + "selected_utility_gain_over_base": -0.9475514180958271, + "selector_regret": 0.024834569543600082, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.461885278016425 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7623409522984086, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0664558708667755, + "selected_utility_gain_over_base": -0.9335441291332245, + "selector_regret": 0.9335441291332245, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5871931326735697, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0387911088764668, + "selected_utility_gain_over_base": -0.9612088911235332, + "selector_regret": 0.9612088911235332, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.01765980271334038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.461885278016425 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.461885278016425 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.461885278016425 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6258640055379121, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1094239205121994, + "selected_utility_gain_over_base": -0.08148449659347534, + "selector_regret": 1.8905760794878006, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5604320834866071, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18476036190986633, + "selected_utility_gain_over_base": -0.006148055195808411, + "selector_regret": 0.02819681167602539, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.461885278016425 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6258640055379121, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12799203395843506, + "selected_utility_gain_over_base": -0.06291638314723969, + "selector_regret": 0.10589073598384857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.461885278016425 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.18325703686238062, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.0306213330477476, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.461885278016425 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.18325703686238062, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 4.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.9949396047741175, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.461885278016425 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.18325703686238062, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.0932139214128256, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.03189693906883312, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.461885278016425 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.024889983439393283, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.461885278016425 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.045483710612303836, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.461885278016425 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2296842211404253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.25659152366583776, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2296842211404253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7208522520964407, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8035469324826213, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7208522520964407, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.461885278016425 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.01069560631150106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.461885278016425 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.425471802912922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.4513022033896152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.425471802912922, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.461885278016425 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.783211815160911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.783211815160911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04792279005050659, + "selected_utility_gain_over_base": -0.13379275798797607, + "selector_regret": 1.9520772099494934, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.783211815160911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.26553206302513555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.26553206302513555, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.26553206302513555, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.461885278016425 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": -0.05953940375790834, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00509834272088483, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03959713288350031, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.461885278016425 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.18621980420068318, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 7.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00509834272088483, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.030729531950782984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.461885278016425 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.1759229406142279, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00509834272088483, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03980091231642291, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.461885278016425 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.730922283654924, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.054690223187208176, + "selected_utility_gain_over_base": -0.028444822877645493, + "selector_regret": 0.03357108309864998, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.461885278016425 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.5715662758164363, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01698315143585205, + "selected_utility_gain_over_base": -0.06615189462900162, + "selector_regret": 0.050428032875061035, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.461885278016425 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.730922283654924, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02323226071894169, + "selected_utility_gain_over_base": -0.05990278534591198, + "selector_regret": 0.05241936258971691, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.461885278016425 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5653215567693661, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11142203211784363, + "selected_utility_gain_over_base": -0.051584646105766296, + "selector_regret": 0.025906652212142944, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.461885278016425 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.558522723271604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13658663630485535, + "selected_utility_gain_over_base": -0.026420041918754578, + "selector_regret": 0.6762891709804535, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.461885278016425 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.558522723271604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19850653409957886, + "selected_utility_gain_over_base": 0.03549985587596893, + "selector_regret": 0.01596677303314209, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.461885278016425 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37033219487548785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.09793626053678911, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37033219487548785, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.5199957201646186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.5199957201646186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 1.5199957201646186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.461885278016425 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.11147982914005515, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.461885278016425 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.15923551786737147, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.461885278016425 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.15923551786737147, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.461885278016425 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.8358427629109161, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 2.6666666666666665, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.7116076741367579, + "selected_utility_gain_over_base": 0.5940145247926315, + "selector_regret": 0.041052653764685, + "success_selector_gap": 0.0, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.4618852780164249 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.6027052046516154, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 1.6666666666666667, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.7338168621063232, + "selected_utility_gain_over_base": 0.6162237127621969, + "selector_regret": 0.6366864119966825, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.4618852780164249 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.5885351499944433, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 0.6666666666666666, + "selected_success": 0.6666666666666666, + "selected_success_gain_over_base": 0.6666666666666666, + "selected_utility": 1.3643324971199036, + "selected_utility_gain_over_base": 1.246739347775777, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.4618852780164249 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.5165554186299326, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 11.25, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.039441852990421467, + "selected_utility_gain_over_base": -0.5337574890145333, + "selector_regret": 1.0117176117637428, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.461885278016425 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.51883899857906, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 8.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5731993420049548, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.4925890024751425, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.461885278016425 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.45152492482904577, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 3.25, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.08047536047524773, + "selected_utility_gain_over_base": -0.4927239815297071, + "selector_regret": 0.9877707070263568, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.461885278016425 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..b1cf9601c82a3f7638e5df185c5a8b5e03c486db --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "predicted_margin": -0.052165254117878486, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 3.466666666666667, + "selected_success": 0.2, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.9498705387115478, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5146939317385356, + "success_selector_gap": 0.4, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.46188527801642487 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.07343996006022091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 6.666666666666667, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.4618852780164249 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.3834765656041302, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 4.16, + "selected_success": 0.3466666666666667, + "selected_success_gain_over_base": -0.013333333333333334, + "selected_utility": 0.7355007326719352, + "selected_utility_gain_over_base": -0.039669728551525625, + "selector_regret": 0.4549120608600788, + "success_selector_gap": 0.22666666666666666, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.46188527801642476 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "predicted_margin": 0.8374506997361121, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 4.555555555555555, + "selected_success": 0.4444444444444444, + "selected_success_gain_over_base": 0.2777777777777778, + "selected_utility": 0.9414158223403825, + "selected_utility_gain_over_base": 0.549011643562052, + "selector_regret": 0.8443655926320288, + "success_selector_gap": 0.4444444444444444, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.46188527801642487 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.36363636363636365, + "execute_generated": 0.36363636363636365, + "fallback_rate": 0.6363636363636364, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "predicted_margin": 0.3693245330816113, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 3.878787878787879, + "selected_success": 0.30303030303030304, + "selected_success_gain_over_base": 0.030303030303030304, + "selected_utility": 0.7855989084099279, + "selected_utility_gain_over_base": 0.009204203883806864, + "selector_regret": 0.3740111264315518, + "success_selector_gap": 0.21212121212121213, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.46188527801642476 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..60b505bc51c453c36566a39c2a80927208368fef --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_regressor` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 40, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.461885` +Feature set: `chart_compat` +Target: `success_weighted_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.3357 | 0.6643 | 0.2780 | 0.4621 | 0.5343 | 0.5343 | 0.1733 | 0.1264 | +| selection | 0.3355 | 0.6645 | 0.2774 | 0.3742 | 0.5419 | 0.5677 | 0.1806 | 0.2065 | +| eval | 0.3403 | 0.6597 | 0.2917 | 0.3264 | 0.5694 | 0.5486 | 0.2014 | 0.2639 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..2706c3e0547971eab692012771ffa1c44982d2e5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,20 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + }, + "source_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..9a1a43296e330334c511801c6d1e76c09183dde9 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.3403 & 0.6597 & 0.2917 & 0.3264 & 0.5694 & 0.5486 & 0.2014 & 0.2639 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..aa7e0811807554c58ce288ec044dbf2d2601bdea --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_regressor +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 40, "max_leaf_nodes": 15, "random_state": 0} +tau=0.461885 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..947085da4f6a5b4b6c6fe72e168cbf7d3ad1386a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test --feature-set chart_compat --selector-chart-feature-mode base_context_obs --target utility_margin --k 16 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f53509d4817b128f9debeab06790c306ec0f5a39 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,15 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_compat +k: 16 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test +seed: 0 +selection_frac: 0.35 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +target: utility_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c001552e076e38f762c1980a9ac2fdd590af6dc4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..baa5577bac49fd223fab1fbf679dd6fa783dc950 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,7470 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.3541666666666667, + "fallback_rate": 0.6458333333333334, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.2847222222222222, + "selected_success_gain_over_base": -0.006944444444444444, + "selected_utility": 0.7162999392102292, + "selector_regret": 0.5429793549620727, + "success_selector_gap": 0.2986111111111111, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "chart_compat", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4548736462093863, + "fallback_rate": 0.5451263537906137, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.5342960288808665, + "proposal_oracle_success": 0.5342960288808665, + "proposal_oracle_utility": 1.1336619752116286, + "selected_success": 0.4620938628158845, + "selected_success_gain_over_base": 0.18411552346570398, + "selected_utility": 1.0263910738352875, + "selector_regret": 0.2268675415909476, + "success_selector_gap": 0.1263537906137184, + "success_support_gap": 0.17328519855595667, + "support_gap": 0.3521794904517649 + }, + "k": 16, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.49032258064516127, + "fallback_rate": 0.5096774193548387, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.44516129032258067, + "selected_success_gain_over_base": 0.16774193548387098, + "selected_utility": 0.9639207383876877, + "selector_regret": 0.2880004003505042, + "success_selector_gap": 0.14193548387096774, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "num_fit_candidates": 4175, + "num_fit_rows": 277, + "num_selection_candidates": 2335, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.08991720318616597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2797080480367002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28355827608242967, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18560038716365618, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20172868695503152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08863633481483887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3352371313866231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4110900404587235, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.023549815639853477, + "selected_utility_gain_over_base": -0.13813611306250095, + "selector_regret": 1.9764501843601465, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3050981300072402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.6362267958435123, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0025064547080546618, + "selected_utility_gain_over_base": 0.0005769530544057488, + "selector_regret": 0.007676919689401984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.507461874569938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1048971638083458, + "selected_utility_gain_over_base": -1.8951028361916542, + "selector_regret": 0.049669139087200165, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042802515015988585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.17492469965849014, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.35986959125133827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09716508537530899, + "selected_utility_gain_over_base": 0.061278779059648514, + "selector_regret": 0.04954756051301956, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22822097403519162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.19016637277131918, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.26403852161187397, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26430287849596273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3988789611600871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0042244791984558105, + "selected_utility_gain_over_base": -0.09860976785421371, + "selector_regret": 0.010953426361083984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34069303977256765, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371194860392053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.2726537010014917, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16043123876023302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05387979745864868, + "selected_utility_gain_over_base": -1.9461202025413513, + "selector_regret": 1.9461202025413513, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.615562919823872, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019754549488425255, + "selected_utility_gain_over_base": -0.07324294187128544, + "selector_regret": 0.02163984440267086, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46618538705348106, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.28297507531186483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.471092511551753, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.4534903350362056, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10685249697353241, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3010393386519421, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244761772076728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.061532557010650635, + "selected_utility_gain_over_base": -0.044937968254089355, + "selector_regret": 0.08252748847007751, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7256808008161852, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14965972304344177, + "selected_utility_gain_over_base": 0.024892255663871765, + "selector_regret": 1.8503402769565582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5566373654374286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.39582323274190295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16138880459399857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30368062376907057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32662525863909614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2255580985362039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5870819789979614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36446911237871227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11833827197551727, + "selected_utility_gain_over_base": -0.07257014513015747, + "selector_regret": 1.8816617280244827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20532758727946646, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1953526650841092, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3589247640152091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20585303592937382, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2830091920285253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1018745030033766, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24778606153341362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3337636013451598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2379598925580104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24443771108596596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09154681464635614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33370414261502274, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3705938518118879, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10141054540872574, + "selected_utility_gain_over_base": -0.06027538329362869, + "selector_regret": 1.8985894545912743, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.7484074094930177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.6387349542040327, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007475614547729492, + "selected_utility_gain_over_base": 0.005546112894080579, + "selector_regret": 0.00242034625262022, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41414798691644433, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13291174173355103, + "selected_utility_gain_over_base": -1.867088258266449, + "selector_regret": 1.867088258266449, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04466194101303914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.17110779360440906, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.4007460177868561, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04821258783340454, + "selected_utility_gain_over_base": 0.012326281517744064, + "selector_regret": 0.027136601507663727, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23290921447763308, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.29761817086705544, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05349260475486517, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.28343513695517897, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23332367249842212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37538333059509166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035802166908979416, + "selected_utility_gain_over_base": -0.06703208014369011, + "selector_regret": 0.001208961009979248, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.268590110511807, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2496823681236075, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.3036913214804951, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17011388186485987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39227093434151067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.023384133353829384, + "selected_utility_gain_over_base": -1.9766158666461706, + "selector_regret": 1.9766158666461706, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.3948568337614319, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08876892179250717, + "selected_utility_gain_over_base": -0.004228569567203522, + "selector_regret": 0.02274090051651001, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5592655513262915, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2525195047503382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.46137979322703987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15205878019332886, + "selected_utility_gain_over_base": 0.102475106716156, + "selector_regret": 1.8479412198066711, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.1768234215359843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11932515699499706, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34125143842352146, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48070996484280953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7407334225672018, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5940825716503084, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3836105423742797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43921922377953376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12788381852052721, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2713166038796752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27481853311866083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21684013213485268, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6097239173416052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21665335322366186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10311308876908776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38798745623031156, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20493799872831076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 10, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2762933949412652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 9, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09743460718887578, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29487688188584205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29522744379276233, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1808325968136315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19710526958727811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08595256684637552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33681729037438607, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35345287441810663, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18884079158306122, + "selected_utility_gain_over_base": 0.027154862880706787, + "selector_regret": 1.8111592084169388, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.2910706981901193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.5766920271631167, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02248450182378292, + "selected_utility_gain_over_base": 0.020555000170134008, + "selector_regret": 0.016367433592677116, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28546890192574864, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.048270193237023314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.14612092709825528, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3319989763272131, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23237107122905468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.1915588212964643, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.30981294473381726, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2531734715082984, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30448894670641713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2722642721291662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2912910141265051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.31737923139011737, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15542103752775793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 15, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.46269040126126526, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.051346004009246826, + "selected_utility_gain_over_base": -0.04165148735046387, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.562175027833504, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.5259115214300562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06632735580205917, + "selected_utility_gain_over_base": -1.9336726441979408, + "selector_regret": 0.05877452343702316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4505066498805008, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06990895420312881, + "selected_utility_gain_over_base": 0.020325280725955963, + "selector_regret": 1.9300910457968712, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5340001574841613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02059849165380001, + "selected_utility_gain_over_base": -0.08391634561121464, + "selector_regret": 0.09250334836542606, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12054968766555163, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2153900468807876, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4884411626074505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7188844098969164, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5877928871421093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04538704454898834, + "selected_utility_gain_over_base": -1.9546129554510117, + "selector_regret": 1.9546129554510117, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3982631871077599, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13169501516378973, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19620741382881715, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3131801253335776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2250806530775492, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5974323487206175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17691531777381897, + "selected_utility_gain_over_base": -0.013993099331855774, + "selector_regret": 0.05696745216846466, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22061568738801626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17894998490871183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4021201653187239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.18125860724557305, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32995590919912554, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3473188188503786, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.49032258064516127, + "fallback_rate": 0.5096774193548387, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.44516129032258067, + "selected_success_gain_over_base": 0.16774193548387098, + "selected_utility": 0.9639207383876877, + "selector_regret": 0.2880004003505042, + "success_selector_gap": 0.14193548387096774, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4194444444444444, + "low": 0.2513888888888889, + "mean": 0.33541666666666675, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33541666666666675, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5208888888888888, + "low": 0.05454545454545454, + "mean": 0.2763232323232323, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2763232323232323, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.38666666666666666, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4194444444444444, + "low": 0.2513888888888889, + "mean": 0.33541666666666675, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.33541666666666675, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5208888888888888, + "low": 0.05454545454545454, + "mean": 0.2763232323232323, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2763232323232323, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.38666666666666666, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7486111111111112, + "low": 0.5805555555555555, + "mean": 0.6645833333333334, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6645833333333334, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.75, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9454545454545455, + "low": 0.47911111111111104, + "mean": 0.7236767676767677, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236767676767677, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.6133333333333333, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.7222222222222222, + "low": 0.5694444444444444, + "mean": 0.6458333333333334, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4430555555555556, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7206060606060607, + "low": 0.2206060606060606, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3404264536488951, + "low": 0.28329633383431224, + "mean": 0.3113264038608651, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3113264038608651, + "per_seed": { + "1073161778/train0": 0.28355827608242967, + "1073161778/train1": 0.3337636013451598, + "1073161778/train2": 0.29522744379276233, + "1113371261/train0": 0.32662525863909614, + "1113371261/train1": 0.27481853311866083, + "1113371261/train2": 0.3131801253335776, + "1151410346/train0": 0.20532758727946646, + "1151410346/train1": 0.21665335322366186, + "1151410346/train2": 0.22061568738801626, + "1261930456/train0": 0.5566373654374286, + "1261930456/train1": 0.5940825716503084, + "1261930456/train2": 0.5877928871421093, + "1324133122/train0": 0.1953526650841092, + "1324133122/train1": 0.10311308876908776, + "1324133122/train2": 0.17894998490871183, + "1324642411/train0": 0.5244761772076728, + "1324642411/train1": 0.48070996484280953, + "1324642411/train2": 0.4884411626074505, + "1375492786/train0": 0.16043123876023302, + "1375492786/train1": 0.17011388186485987, + "1375492786/train2": 0.15542103752775793, + "141984615/train0": 0.3988789611600871, + "141984615/train1": 0.37538333059509166, + "141984615/train2": 0.30448894670641713, + "1461069713/train0": 0.3050981300072402, + "1461069713/train1": 0.7484074094930177, + "1461069713/train2": 0.2910706981901193, + "1480856810/train0": 0.29221027147425893, + "1480856810/train1": 0.43921922377953376, + "1480856810/train2": 0.29221027147425893, + "1509980664/train0": 0.5870819789979614, + "1509980664/train1": 0.6097239173416052, + "1509980664/train2": 0.5974323487206175, + "1522272686/train0": 0.08863633481483887, + "1522272686/train1": 0.09154681464635614, + "1522272686/train2": 0.08595256684637552, + "1549324805/train0": 0.4534903350362056, + "1549324805/train1": 0.1768234215359843, + "1549324805/train2": 0.5340001574841613, + "1578990937/train0": 0.2255580985362039, + "1578990937/train1": 0.21684013213485268, + "1578990937/train2": 0.2250806530775492, + "1747178030/train0": 0.3352371313866231, + "1747178030/train1": 0.33370414261502274, + "1747178030/train2": 0.33681729037438607, + "1790188787/train0": 0.7256808008161852, + "1790188787/train1": 0.7407334225672018, + "1790188787/train2": 0.7188844098969164, + "1822210471/train0": 0.10685249697353241, + "1822210471/train1": 0.11932515699499706, + "1822210471/train2": 0.12054968766555163, + "1845489013/train0": 0.22822097403519162, + "1845489013/train1": 0.23290921447763308, + "1845489013/train2": 0.23237107122905468, + "1999022574/train0": 0.2830091920285253, + "1999022574/train1": 0.2762933949412652, + "1999022574/train2": 0.32995590919912554, + "2012211287/train0": 0.08991720318616597, + "2012211287/train1": 0.1018745030033766, + "2012211287/train2": 0.09743460718887578, + "2038594191/train0": 0.41980384640853585, + "2038594191/train1": 0.39227093434151067, + "2038594191/train2": 0.41980384640853585, + "2054899126/train0": 0.48682363081170954, + "2054899126/train1": 0.4867067691213679, + "2054899126/train2": 0.4238219648796263, + "2061635888/train0": 0.16138880459399857, + "2061635888/train1": 0.12788381852052721, + "2061635888/train2": 0.13169501516378973, + "211245842/train0": 0.36446911237871227, + "211245842/train1": 0.3818735579959133, + "211245842/train2": 0.3818735579959133, + "278008524/train0": 0.4154810514015457, + "278008524/train1": 0.431062905506948, + "278008524/train2": 0.39125281310385696, + "307068879/train0": 0.2797080480367002, + "307068879/train1": 0.24778606153341362, + "307068879/train2": 0.29487688188584205, + "412102563/train0": 0.20585303592937382, + "412102563/train1": 0.20493799872831076, + "412102563/train2": 0.18125860724557305, + "440577711/train0": 0.1872847414980603, + "440577711/train1": 0.1872847414980603, + "440577711/train2": 0.1872847414980603, + "441046872/train0": 0.20172868695503152, + "441046872/train1": 0.24443771108596596, + "441046872/train2": 0.19710526958727811, + "509430392/train0": 0.39582323274190295, + "509430392/train1": 0.3836105423742797, + "509430392/train2": 0.3982631871077599, + "585904130/train0": 0.26430287849596273, + "585904130/train1": 0.23332367249842212, + "585904130/train2": 0.2531734715082984, + "608016290/train0": 0.18560038716365618, + "608016290/train1": 0.2379598925580104, + "608016290/train2": 0.1808325968136315, + "626486813/train0": 0.41319658430741574, + "626486813/train1": 0.46817656253554407, + "626486813/train2": 0.3841254242297905, + "636680137/train0": 0.5371194860392053, + "636680137/train1": 0.2496823681236075, + "636680137/train2": 0.2912910141265051, + "655884605/train0": 0.30368062376907057, + "655884605/train1": 0.2713166038796752, + "655884605/train2": 0.19620741382881715, + "672328262/train0": 0.042802515015988585, + "672328262/train1": 0.04466194101303914, + "672328262/train2": 0.048270193237023314, + "701667486/train0": 0.3589247640152091, + "701667486/train1": 0.38798745623031156, + "701667486/train2": 0.4021201653187239, + "747260437/train0": 0.28297507531186483, + "747260437/train1": 0.2525195047503382, + "747260437/train2": 0.5259115214300562, + "750448296/train0": 0.4190984327494606, + "750448296/train1": 0.33489946089892014, + "750448296/train2": 0.31678157342373975, + "767747424/train0": 0.28067674068264487, + "767747424/train1": 0.28220702596299274, + "767747424/train2": 0.28169149441007413 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3813259240033606, + "low": 0.16290313710457643, + "mean": 0.27556930577634775, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.27556930577634775, + "per_task": { + "LiftPegUpright-v1": 0.21812074468121995, + "PickCube-v1": 0.08871190543585684, + "PullCube-v1": 0.33025922453409173, + "PushCube-v1": 0.4544794286452365, + "StackCube-v1": 0.2862752255853337 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.34431658455017794, + "low": 0.29466522012600893, + "mean": 0.3189937561563403, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6722222222222223, + "low": 0.5055555555555555, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7430303030303029, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4930555555555556, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4116970675635254, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5246927151765788, + "low": 0.5848668839583795, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.360821089959548, + "low": 1.0729441113071516, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 5.716666666666667, + "low": 4.227083333333334, + "mean": 4.927083333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 4.927083333333333, + "per_seed": { + "1073161778/train0": 6.0, + "1073161778/train1": 7.0, + "1073161778/train2": 3.0, + "1113371261/train0": 0.0, + "1113371261/train1": 4.0, + "1113371261/train2": 2.0, + "1151410346/train0": 7.0, + "1151410346/train1": 9.0, + "1151410346/train2": 10.0, + "1261930456/train0": 5.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 5.0, + "1324133122/train2": 7.0, + "1324642411/train0": 3.0, + "1324642411/train1": 9.0, + "1324642411/train2": 3.0, + "1375492786/train0": 15.0, + "1375492786/train1": 10.0, + "1375492786/train2": 15.0, + "141984615/train0": 1.0, + "141984615/train1": 6.0, + "141984615/train2": 4.0, + "1461069713/train0": 2.0, + "1461069713/train1": 0.0, + "1461069713/train2": 5.0, + "1480856810/train0": 2.0, + "1480856810/train1": 13.0, + "1480856810/train2": 1.0, + "1509980664/train0": 10.0, + "1509980664/train1": 4.0, + "1509980664/train2": 12.0, + "1522272686/train0": 8.0, + "1522272686/train1": 7.0, + "1522272686/train2": 8.0, + "1549324805/train0": 0.0, + "1549324805/train1": 2.0, + "1549324805/train2": 3.0, + "1578990937/train0": 15.0, + "1578990937/train1": 12.0, + "1578990937/train2": 0.0, + "1747178030/train0": 5.0, + "1747178030/train1": 9.0, + "1747178030/train2": 0.0, + "1790188787/train0": 9.0, + "1790188787/train1": 6.0, + "1790188787/train2": 0.0, + "1822210471/train0": 8.0, + "1822210471/train1": 5.0, + "1822210471/train2": 0.0, + "1845489013/train0": 5.0, + "1845489013/train1": 7.0, + "1845489013/train2": 8.0, + "1999022574/train0": 11.0, + "1999022574/train1": 9.0, + "1999022574/train2": 6.0, + "2012211287/train0": 0.0, + "2012211287/train1": 5.0, + "2012211287/train2": 1.0, + "2038594191/train0": 3.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 5.0, + "2061635888/train1": 6.0, + "2061635888/train2": 4.0, + "211245842/train0": 7.0, + "211245842/train1": 13.0, + "211245842/train2": 0.0, + "278008524/train0": 3.0, + "278008524/train1": 1.0, + "278008524/train2": 11.5, + "307068879/train0": 2.0, + "307068879/train1": 12.0, + "307068879/train2": 0.0, + "412102563/train0": 6.0, + "412102563/train1": 10.0, + "412102563/train2": 3.0, + "440577711/train0": 3.0, + "440577711/train1": 5.0, + "440577711/train2": 5.0, + "441046872/train0": 10.0, + "441046872/train1": 3.0, + "441046872/train2": 0.0, + "509430392/train0": 14.0, + "509430392/train1": 2.0, + "509430392/train2": 1.0, + "585904130/train0": 14.0, + "585904130/train1": 7.0, + "585904130/train2": 4.0, + "608016290/train0": 3.0, + "608016290/train1": 8.0, + "608016290/train2": 15.0, + "626486813/train0": 0.0, + "626486813/train1": 0.5, + "626486813/train2": 1.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 2.0, + "655884605/train2": 8.0, + "672328262/train0": 8.0, + "672328262/train1": 1.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 10.0, + "701667486/train2": 8.0, + "747260437/train0": 0.0, + "747260437/train1": 8.0, + "747260437/train2": 1.0, + "750448296/train0": 4.0, + "750448296/train1": 3.0, + "750448296/train2": 2.0, + "767747424/train0": 7.0, + "767747424/train1": 2.0, + "767747424/train2": 7.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 6.771717171717173, + "low": 4.363555555555555, + "mean": 5.475717171717173, + "n": 5, + "num_samples": 1000 + }, + "mean": 5.475717171717173, + "per_task": { + "LiftPegUpright-v1": 5.0, + "PickCube-v1": 7.666666666666667, + "PullCube-v1": 3.8533333333333335, + "PushCube-v1": 4.555555555555555, + "StackCube-v1": 6.303030303030303 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 5.409722222222222, + "low": 4.020833333333333, + "mean": 4.701388888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3875, + "low": 0.21875, + "mean": 0.3020833333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3020833333333333, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.29943434343434344, + "low": 0.10060606060606062, + "mean": 0.2174949494949495, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2174949494949495, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.30666666666666664, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2152777777777778, + "mean": 0.2847222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.041666666666666664, + "low": -0.05416666666666667, + "mean": -0.004166666666666667, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.004166666666666667, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.0, + "2038594191/train1": -1.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.06666666666666667, + "low": -0.02593939393939394, + "mean": 0.017616161616161613, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.017616161616161613, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.05333333333333334, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.034722222222222224, + "low": -0.05555555555555555, + "mean": -0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9239329974577736, + "low": 0.6214197847254884, + "mean": 0.7678461809070238, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7678461809070238, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 0.04538704454898834, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.061532557010650635, + "1324642411/train1": 0.1280023753643036, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.0042244791984558105, + "141984615/train1": 0.035802166908979416, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.009687979705631733, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.20681259036064148, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 2.0, + "1509980664/train1": 0.21804393827915192, + "1509980664/train2": 0.8189421892166138, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.02059849165380001, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.14965972304344177, + "1790188787/train1": 0.13741976022720337, + "1790188787/train2": 2.0, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 0.05387979745864868, + "2038594191/train1": 0.023384133353829384, + "2038594191/train2": 2.0, + "2054899126/train0": 1.052448581904173, + "2054899126/train1": 1.0664558708667755, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.11833827197551727, + "211245842/train1": 0.17156320810317993, + "211245842/train2": 0.17691531777381897, + "278008524/train0": 1.0485825426876545, + "278008524/train1": 0.1001356840133667, + "278008524/train2": 0.052897630259394646, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.12097799777984619, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.005386819248087704, + "626486813/train1": 0.00787139916792512, + "626486813/train2": 0.015375842805951834, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 0.06632735580205917, + "750448296/train0": 0.09317883538703124, + "750448296/train1": 0.11618362615505855, + "750448296/train2": 0.1037093202273051, + "767747424/train0": 0.5386653137393296, + "767747424/train1": 0.5581304961815476, + "767747424/train2": 0.5799880577251315 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8515048929266256, + "low": 0.6142033413514081, + "mean": 0.7215833358311028, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7215833358311028, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.666174189314867, + "PushCube-v1": 0.6184621560904715, + "StackCube-v1": 0.7894506924080126 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8554598261252977, + "low": 0.5858313379317729, + "mean": 0.7162999392102292, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.06609533377680539, + "low": -0.11685185874133215, + "mean": -0.02227188139077043, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.02227188139077043, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.9546129554510117, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -0.044937968254089355, + "1324642411/train1": 0.0215318500995636, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.09860976785421371, + "141984615/train1": -0.06703208014369011, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": -0.08421508502215147, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.11977851390838623, + "1480856810/train2": 0.0, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": -0.6153660863637924, + "1509980664/train2": -0.014467835426330566, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": 0.0, + "1549324805/train2": -0.08391634561121464, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.024892255663871765, + "1790188787/train1": 0.012652292847633362, + "1790188787/train2": 1.87523253262043, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": -1.9461202025413513, + "2038594191/train1": -1.9766158666461706, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9475514180958271, + "2054899126/train1": -0.9335441291332245, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": -0.07257014513015747, + "211245842/train1": -0.019345209002494812, + "211245842/train2": -0.013993099331855774, + "278008524/train0": 1.0058475527912378, + "278008524/train1": 0.057400694116950035, + "278008524/train2": 0.010162640362977982, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.12097777426242828, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0002884765272028744, + "626486813/train1": 0.0027730564470402896, + "626486813/train2": 0.010277500085067004, + "636680137/train0": -0.027042903006076813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.9336726441979408, + "750448296/train0": -0.024414313957095146, + "750448296/train1": -0.0014095231890678406, + "750448296/train2": -0.013883829116821289, + "767747424/train0": -0.03453402826562524, + "767747424/train1": -0.015068845823407173, + "767747424/train2": 0.006788715720176697 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1356347863872846, + "low": -0.06278656556877787, + "mean": 0.026023538657087796, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.026023538657087796, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.10899627190859368, + "PushCube-v1": 0.226057977312141, + "StackCube-v1": 0.013055987881891655 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.05458070211009019, + "low": -0.12055642578343395, + "mean": -0.025519647232108075, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6801490995260085, + "low": 0.40305140065320294, + "mean": 0.5353515408005073, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5353515408005073, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.9546129554510117, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.08252748847007751, + "1324642411/train1": 1.8719976246356964, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.010953426361083984, + "141984615/train1": 0.001208961009979248, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.00689703319221735, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.7931874096393585, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 0.0, + "1509980664/train1": 1.781956061720848, + "1509980664/train2": 1.1810578107833862, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.09250334836542606, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.8503402769565582, + "1790188787/train1": 1.8625802397727966, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 1.9461202025413513, + "2038594191/train1": 1.9766158666461706, + "2038594191/train2": 0.0, + "2054899126/train0": 0.024834569543600082, + "2054899126/train1": 0.9335441291332245, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8816617280244827, + "211245842/train1": 0.04139396548271179, + "211245842/train2": 0.05696745216846466, + "278008524/train0": 0.02477378025650978, + "278008524/train1": 0.9375389106571674, + "278008524/train2": 1.0830512810498476, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 1.3000448346138, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8790220022201538, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.03930865635629743, + "626486813/train1": 0.027956475503742695, + "626486813/train2": 0.029523412231355906, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.6498691290616989, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.05877452343702316, + "750448296/train0": 0.6594814925144116, + "750448296/train1": 1.2543196479479473, + "750448296/train2": 1.246739347775777, + "767747424/train0": 0.5124941510148346, + "767747424/train1": 0.5076578482985497, + "767747424/train2": 0.48825800977647305 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8738215418463066, + "low": 0.17712627903981643, + "mean": 0.5098326246412377, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5098326246412377, + "per_task": { + "LiftPegUpright-v1": 0.5146939317385356, + "PickCube-v1": 0.0, + "PullCube-v1": 0.49621246912516653, + "PushCube-v1": 1.1673192588819399, + "StackCube-v1": 0.37093746346054657 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.6691991107056512, + "low": 0.41660342318290433, + "mean": 0.5429793549620727, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38333333333333336, + "low": 0.225, + "mean": 0.29930555555555555, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.29930555555555555, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4680000000000001, + "low": 0.12242424242424244, + "mean": 0.2953131313131313, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2953131313131313, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.25333333333333335, + "PushCube-v1": 0.6111111111111112, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3680555555555556, + "low": 0.2222222222222222, + "mean": 0.2986111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30416666666666664, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7191919191919192, + "low": 0.13511111111111113, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2638888888888889, + "low": 0.1388888888888889, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5434736852068454, + "low": 0.2965679521788843, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.330930626693398, + "low": 0.24626440485649642, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.485598717091812, + "low": 0.272009730946997, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.34731881885037813, + "low": 0.34731881885037813, + "mean": 0.34731881885037813, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.34731881885037813, + "per_seed": { + "1073161778/train0": 0.3473188188503786, + "1073161778/train1": 0.3473188188503786, + "1073161778/train2": 0.3473188188503786, + "1113371261/train0": 0.3473188188503786, + "1113371261/train1": 0.3473188188503786, + "1113371261/train2": 0.3473188188503786, + "1151410346/train0": 0.3473188188503786, + "1151410346/train1": 0.3473188188503786, + "1151410346/train2": 0.3473188188503786, + "1261930456/train0": 0.3473188188503786, + "1261930456/train1": 0.3473188188503786, + "1261930456/train2": 0.3473188188503786, + "1324133122/train0": 0.3473188188503786, + "1324133122/train1": 0.3473188188503786, + "1324133122/train2": 0.3473188188503786, + "1324642411/train0": 0.3473188188503786, + "1324642411/train1": 0.3473188188503786, + "1324642411/train2": 0.3473188188503786, + "1375492786/train0": 0.3473188188503786, + "1375492786/train1": 0.3473188188503786, + "1375492786/train2": 0.3473188188503786, + "141984615/train0": 0.3473188188503786, + "141984615/train1": 0.3473188188503786, + "141984615/train2": 0.3473188188503786, + "1461069713/train0": 0.3473188188503786, + "1461069713/train1": 0.3473188188503786, + "1461069713/train2": 0.3473188188503786, + "1480856810/train0": 0.3473188188503786, + "1480856810/train1": 0.3473188188503786, + "1480856810/train2": 0.3473188188503786, + "1509980664/train0": 0.3473188188503786, + "1509980664/train1": 0.3473188188503786, + "1509980664/train2": 0.3473188188503786, + "1522272686/train0": 0.3473188188503786, + "1522272686/train1": 0.3473188188503786, + "1522272686/train2": 0.3473188188503786, + "1549324805/train0": 0.3473188188503786, + "1549324805/train1": 0.3473188188503786, + "1549324805/train2": 0.3473188188503786, + "1578990937/train0": 0.3473188188503786, + "1578990937/train1": 0.3473188188503786, + "1578990937/train2": 0.3473188188503786, + "1747178030/train0": 0.3473188188503786, + "1747178030/train1": 0.3473188188503786, + "1747178030/train2": 0.3473188188503786, + "1790188787/train0": 0.3473188188503786, + "1790188787/train1": 0.3473188188503786, + "1790188787/train2": 0.3473188188503786, + "1822210471/train0": 0.3473188188503786, + "1822210471/train1": 0.3473188188503786, + "1822210471/train2": 0.3473188188503786, + "1845489013/train0": 0.3473188188503786, + "1845489013/train1": 0.3473188188503786, + "1845489013/train2": 0.3473188188503786, + "1999022574/train0": 0.3473188188503786, + "1999022574/train1": 0.3473188188503786, + "1999022574/train2": 0.3473188188503786, + "2012211287/train0": 0.3473188188503786, + "2012211287/train1": 0.3473188188503786, + "2012211287/train2": 0.3473188188503786, + "2038594191/train0": 0.3473188188503786, + "2038594191/train1": 0.3473188188503786, + "2038594191/train2": 0.3473188188503786, + "2054899126/train0": 0.3473188188503786, + "2054899126/train1": 0.3473188188503786, + "2054899126/train2": 0.3473188188503786, + "2061635888/train0": 0.3473188188503786, + "2061635888/train1": 0.3473188188503786, + "2061635888/train2": 0.3473188188503786, + "211245842/train0": 0.3473188188503786, + "211245842/train1": 0.3473188188503786, + "211245842/train2": 0.3473188188503786, + "278008524/train0": 0.3473188188503786, + "278008524/train1": 0.3473188188503786, + "278008524/train2": 0.3473188188503786, + "307068879/train0": 0.3473188188503786, + "307068879/train1": 0.3473188188503786, + "307068879/train2": 0.3473188188503786, + "412102563/train0": 0.3473188188503786, + "412102563/train1": 0.3473188188503786, + "412102563/train2": 0.3473188188503786, + "440577711/train0": 0.3473188188503786, + "440577711/train1": 0.3473188188503786, + "440577711/train2": 0.3473188188503786, + "441046872/train0": 0.3473188188503786, + "441046872/train1": 0.3473188188503786, + "441046872/train2": 0.3473188188503786, + "509430392/train0": 0.3473188188503786, + "509430392/train1": 0.3473188188503786, + "509430392/train2": 0.3473188188503786, + "585904130/train0": 0.3473188188503786, + "585904130/train1": 0.3473188188503786, + "585904130/train2": 0.3473188188503786, + "608016290/train0": 0.3473188188503786, + "608016290/train1": 0.3473188188503786, + "608016290/train2": 0.3473188188503786, + "626486813/train0": 0.3473188188503786, + "626486813/train1": 0.3473188188503786, + "626486813/train2": 0.3473188188503786, + "636680137/train0": 0.3473188188503786, + "636680137/train1": 0.3473188188503786, + "636680137/train2": 0.3473188188503786, + "655884605/train0": 0.3473188188503786, + "655884605/train1": 0.3473188188503786, + "655884605/train2": 0.3473188188503786, + "672328262/train0": 0.3473188188503786, + "672328262/train1": 0.3473188188503786, + "672328262/train2": 0.3473188188503786, + "701667486/train0": 0.3473188188503786, + "701667486/train1": 0.3473188188503786, + "701667486/train2": 0.3473188188503786, + "747260437/train0": 0.3473188188503786, + "747260437/train1": 0.3473188188503786, + "747260437/train2": 0.3473188188503786, + "750448296/train0": 0.3473188188503786, + "750448296/train1": 0.3473188188503786, + "750448296/train2": 0.3473188188503786, + "767747424/train0": 0.3473188188503786, + "767747424/train1": 0.3473188188503786, + "767747424/train2": 0.3473188188503786 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.34731881885037874, + "low": 0.34731881885037846, + "mean": 0.3473188188503786, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3473188188503786, + "per_task": { + "LiftPegUpright-v1": 0.34731881885037846, + "PickCube-v1": 0.3473188188503786, + "PullCube-v1": 0.34731881885037885, + "PushCube-v1": 0.3473188188503784, + "StackCube-v1": 0.3473188188503786 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.3473188188503777, + "low": 0.3473188188503777, + "mean": 0.3473188188503777, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": 0.3473188188503786 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..0bf7f4dae4c0ac2069e096d6ba0e5ffe11eb84c7 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28355827608242967, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3337636013451598, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29522744379276233, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32662525863909614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.3473188188503786 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27481853311866083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.3473188188503786 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3131801253335776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.3473188188503786 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20532758727946646, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.3473188188503786 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21665335322366186, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22061568738801626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.3473188188503786 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5566373654374286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5940825716503084, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5877928871421093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04538704454898834, + "selected_utility_gain_over_base": -1.9546129554510117, + "selector_regret": 1.9546129554510117, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1953526650841092, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.3473188188503786 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10311308876908776, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.3473188188503786 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17894998490871183, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.3473188188503786 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244761772076728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.061532557010650635, + "selected_utility_gain_over_base": -0.044937968254089355, + "selector_regret": 0.08252748847007751, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.3473188188503786 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48070996484280953, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1280023753643036, + "selected_utility_gain_over_base": 0.0215318500995636, + "selector_regret": 1.8719976246356964, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4884411626074505, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16043123876023302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17011388186485987, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15542103752775793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3988789611600871, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0042244791984558105, + "selected_utility_gain_over_base": -0.09860976785421371, + "selector_regret": 0.010953426361083984, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.3473188188503786 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37538333059509166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035802166908979416, + "selected_utility_gain_over_base": -0.06703208014369011, + "selector_regret": 0.001208961009979248, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.3473188188503786 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30448894670641713, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.3473188188503786 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3050981300072402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.3473188188503786 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.7484074094930177, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009687979705631733, + "selected_utility_gain_over_base": -0.08421508502215147, + "selector_regret": 0.00689703319221735, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.3473188188503786 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.2910706981901193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.3473188188503786 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43921922377953376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20681259036064148, + "selected_utility_gain_over_base": 0.11977851390838623, + "selector_regret": 1.7931874096393585, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.29221027147425893, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5870819789979614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6097239173416052, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21804393827915192, + "selected_utility_gain_over_base": -0.6153660863637924, + "selector_regret": 1.781956061720848, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5974323487206175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8189421892166138, + "selected_utility_gain_over_base": -0.014467835426330566, + "selector_regret": 1.1810578107833862, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08863633481483887, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.3473188188503786 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09154681464635614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.3473188188503786 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08595256684637552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.3473188188503786 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.4534903350362056, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.3473188188503786 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.1768234215359843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.3473188188503786 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5340001574841613, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02059849165380001, + "selected_utility_gain_over_base": -0.08391634561121464, + "selector_regret": 0.09250334836542606, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.3473188188503786 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2255580985362039, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.21684013213485268, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2250806530775492, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3352371313866231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33370414261502274, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33681729037438607, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7256808008161852, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14965972304344177, + "selected_utility_gain_over_base": 0.024892255663871765, + "selector_regret": 1.8503402769565582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7407334225672018, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13741976022720337, + "selected_utility_gain_over_base": 0.012652292847633362, + "selector_regret": 1.8625802397727966, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7188844098969164, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.10685249697353241, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11932515699499706, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12054968766555163, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22822097403519162, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23290921447763308, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23237107122905468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2830091920285253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2762933949412652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 9.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32995590919912554, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.08991720318616597, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1018745030033766, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09743460718887578, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05387979745864868, + "selected_utility_gain_over_base": -1.9461202025413513, + "selector_regret": 1.9461202025413513, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39227093434151067, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.023384133353829384, + "selected_utility_gain_over_base": -1.9766158666461706, + "selector_regret": 1.9766158666461706, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.41980384640853585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.48682363081170954, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.052448581904173, + "selected_utility_gain_over_base": -0.9475514180958271, + "selector_regret": 0.024834569543600082, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.3473188188503786 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4867067691213679, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0664558708667755, + "selected_utility_gain_over_base": -0.9335441291332245, + "selector_regret": 0.9335441291332245, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4238219648796263, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16138880459399857, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.3473188188503786 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.12788381852052721, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.3473188188503786 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13169501516378973, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.3473188188503786 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36446911237871227, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11833827197551727, + "selected_utility_gain_over_base": -0.07257014513015747, + "selector_regret": 1.8816617280244827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17156320810317993, + "selected_utility_gain_over_base": -0.019345209002494812, + "selector_regret": 0.04139396548271179, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.3473188188503786 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3818735579959133, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17691531777381897, + "selected_utility_gain_over_base": -0.013993099331855774, + "selector_regret": 0.05696745216846466, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.3473188188503786 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.4154810514015457, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 3.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0485825426876545, + "selected_utility_gain_over_base": 1.0058475527912378, + "selector_regret": 0.02477378025650978, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.3473188188503786 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.431062905506948, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1001356840133667, + "selected_utility_gain_over_base": 0.057400694116950035, + "selector_regret": 0.9375389106571674, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.3473188188503786 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.39125281310385696, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 11.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.052897630259394646, + "selected_utility_gain_over_base": 0.010162640362977982, + "selector_regret": 1.0830512810498476, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2797080480367002, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.3473188188503786 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24778606153341362, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.3473188188503786 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29487688188584205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.3473188188503786 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20585303592937382, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20493799872831076, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.18125860724557305, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1872847414980603, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20172868695503152, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24443771108596596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.3473188188503786 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19710526958727811, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.3473188188503786 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.39582323274190295, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3836105423742797, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3982631871077599, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.3473188188503786 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26430287849596273, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23332367249842212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2531734715082984, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18560038716365618, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2379598925580104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1808325968136315, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.3473188188503786 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.41319658430741574, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.005386819248087704, + "selected_utility_gain_over_base": 0.0002884765272028744, + "selector_regret": 0.03930865635629743, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.3473188188503786 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.46817656253554407, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00787139916792512, + "selected_utility_gain_over_base": 0.0027730564470402896, + "selector_regret": 0.027956475503742695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.3473188188503786 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.3841254242297905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015375842805951834, + "selected_utility_gain_over_base": 0.010277500085067004, + "selector_regret": 0.029523412231355906, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.3473188188503786 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371194860392053, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.3473188188503786 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2496823681236075, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.3473188188503786 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.2912910141265051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.3473188188503786 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30368062376907057, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.3473188188503786 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2713166038796752, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6498691290616989, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.3473188188503786 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19620741382881715, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.3473188188503786 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042802515015988585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04466194101303914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.048270193237023314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3589247640152091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38798745623031156, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 10.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4021201653187239, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3473188188503786 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.28297507531186483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.3473188188503786 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2525195047503382, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.3473188188503786 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.5259115214300562, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06632735580205917, + "selected_utility_gain_over_base": -1.9336726441979408, + "selector_regret": 0.05877452343702316, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.3473188188503786 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.4190984327494606, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09317883538703124, + "selected_utility_gain_over_base": -0.024414313957095146, + "selector_regret": 0.6594814925144116, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.3473188188503786 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.33489946089892014, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11618362615505855, + "selected_utility_gain_over_base": -0.0014095231890678406, + "selector_regret": 1.2543196479479473, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.3473188188503786 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.31678157342373975, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1037093202273051, + "selected_utility_gain_over_base": -0.013883829116821289, + "selector_regret": 1.246739347775777, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.3473188188503786 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28067674068264487, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 7.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5386653137393296, + "selected_utility_gain_over_base": -0.03453402826562524, + "selector_regret": 0.5124941510148346, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.3473188188503786 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28220702596299274, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 2.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5581304961815476, + "selected_utility_gain_over_base": -0.015068845823407173, + "selector_regret": 0.5076578482985497, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.3473188188503786 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.28169149441007413, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 7.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5799880577251315, + "selected_utility_gain_over_base": 0.006788715720176697, + "selector_regret": 0.48825800977647305, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.3473188188503786 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..a11dc53387354f6dc7e42837a1c83efaa31679e1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "predicted_margin": 0.21812074468121995, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 5.0, + "selected_success": 0.2, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.9498705387115478, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5146939317385356, + "success_selector_gap": 0.4, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.34731881885037846 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08871190543585684, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 7.666666666666667, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.3473188188503786 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.38666666666666666, + "execute_generated": 0.38666666666666666, + "fallback_rate": 0.6133333333333333, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.33025922453409173, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 3.8533333333333335, + "selected_success": 0.30666666666666664, + "selected_success_gain_over_base": -0.05333333333333334, + "selected_utility": 0.666174189314867, + "selected_utility_gain_over_base": -0.10899627190859368, + "selector_regret": 0.49621246912516653, + "success_selector_gap": 0.25333333333333335, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.34731881885037885 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.7222222222222222, + "execute_generated": 0.7222222222222222, + "fallback_rate": 0.2777777777777778, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "predicted_margin": 0.4544794286452365, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 4.555555555555555, + "selected_success": 0.2777777777777778, + "selected_success_gain_over_base": 0.1111111111111111, + "selected_utility": 0.6184621560904715, + "selected_utility_gain_over_base": 0.226057977312141, + "selector_regret": 1.1673192588819399, + "success_selector_gap": 0.6111111111111112, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.3473188188503784 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.2727272727272727, + "execute_generated": 0.2727272727272727, + "fallback_rate": 0.7272727272727273, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "predicted_margin": 0.2862752255853337, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 6.303030303030303, + "selected_success": 0.30303030303030304, + "selected_success_gain_over_base": 0.030303030303030304, + "selected_utility": 0.7894506924080126, + "selected_utility_gain_over_base": 0.013055987881891655, + "selector_regret": 0.37093746346054657, + "success_selector_gap": 0.21212121212121213, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.3473188188503786 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..00411358a99034504340acb7020996363f91bec4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.347319` +Feature set: `chart_compat` +Target: `utility_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4549 | 0.5451 | 0.2780 | 0.4621 | 0.5343 | 0.5343 | 0.1733 | 0.1264 | +| selection | 0.4903 | 0.5097 | 0.2774 | 0.4452 | 0.5419 | 0.5677 | 0.1806 | 0.1419 | +| eval | 0.3542 | 0.6458 | 0.2917 | 0.2847 | 0.5694 | 0.5486 | 0.2014 | 0.2986 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..2706c3e0547971eab692012771ffa1c44982d2e5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,20 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + }, + "source_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..a3d945b1021ea4fa647f2f4f5760bbc06c826384 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.3542 & 0.6458 & 0.2917 & 0.2847 & 0.5694 & 0.5486 & 0.2014 & 0.2986 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..601d7f3b02f91084eb580b305aaa600dddb73f9f --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0} +tau=0.347319 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..da0361a3864bd1edf868ced06600b0eb8b2d2e00 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --source-index data/cil_charts_rgb_refs/train/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test --feature-set chart_compat --selector-chart-feature-mode base_context_obs --target utility_margin --model-types hgb_regressor,rf_regressor --k 16 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..010e6107c120c03990178a1eade7dbe4b43e577d --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/config.yaml @@ -0,0 +1,15 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: chart_compat +k: 16 +model_types: hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test +seed: 0 +selection_frac: 0.35 +selector_chart_feature_mode: base_context_obs +source_index: data/cil_charts_rgb_refs/train/index.json +target: utility_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..e65c158c17e61aaf10422563d2ffdd8922735e6a --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/data_hash.txt @@ -0,0 +1,7 @@ +{ + "calibration_input": "d79e275b896eb98caa128470d4614b76960cab12407faa157c583224be7abe80", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "541d5a928da5f7ee04ca3dea570e04208830dffde431edfe5b5e38b692634045", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36", + "source_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c001552e076e38f762c1980a9ac2fdd590af6dc4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..c959b9e4eb66a40342a36dd39c739c10b0056837 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics.json @@ -0,0 +1,7469 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_k16_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.4791666666666667, + "fallback_rate": 0.5208333333333334, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5486111111111112, + "proposal_oracle_success": 0.5694444444444444, + "proposal_oracle_utility": 1.2159999172436073, + "selected_success": 0.3055555555555556, + "selected_success_gain_over_base": 0.013888888888888888, + "selected_utility": 0.7636942816805257, + "selector_regret": 0.4682932821221281, + "success_selector_gap": 0.2638888888888889, + "success_support_gap": 0.2013888888888889, + "support_gap": 0.3761285081272945 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "source_chart_feature_found", + "target_chart_feature_norm", + "source_chart_feature_norm", + "target_source_chart_cosine", + "target_source_chart_rms", + "target_source_chart_linf", + "target_source_chart_absmean", + "target_source_chart_dot_per_dim", + "target_base_action_norm", + "source_base_action_norm", + "target_source_base_cosine", + "target_source_base_rms", + "target_source_obs_available", + "target_source_obs_cosine", + "target_source_obs_rms", + "target_obs_norm", + "source_obs_norm", + "target_source_obj_available", + "target_source_obj_cosine", + "target_source_obj_rms", + "target_obj_norm", + "source_obj_norm" + ], + "feature_set": "chart_compat", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4223826714801444, + "fallback_rate": 0.5776173285198556, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.5342960288808665, + "proposal_oracle_success": 0.5342960288808665, + "proposal_oracle_utility": 1.1336619752116286, + "selected_success": 0.4332129963898917, + "selected_success_gain_over_base": 0.1552346570397112, + "selected_utility": 0.968536555992605, + "selector_regret": 0.28316962515944294, + "success_selector_gap": 0.1552346570397112, + "success_support_gap": 0.17328519855595667, + "support_gap": 0.3521794904517649 + }, + "k": 16, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.4645161290322581, + "fallback_rate": 0.535483870967742, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.38064516129032255, + "selected_success_gain_over_base": 0.1032258064516129, + "selected_utility": 0.8593619711727144, + "selector_regret": 0.36662355193928364, + "success_selector_gap": 0.1935483870967742, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "model_types": [ + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 2115, + "num_eval_rows": 144, + "num_fit_candidates": 4175, + "num_fit_rows": 277, + "num_selection_candidates": 2335, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.46177220122772705, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.084742699278881, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.042734309931976376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.2053140291796788, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.034590442020628026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "441046872/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2553463076904834, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17566809733357036, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030047813430428505, + "selected_utility_gain_over_base": -0.13163811527192593, + "selector_regret": 1.9699521865695715, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.003297545598687419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.3453573992092333, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.010183374397456646, + "seed": "626486813/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007142305374145508, + "selected_utility_gain_over_base": 0.005212803720496595, + "selector_regret": 0.003041069023311138, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24323733616620302, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.42663931928548154, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1048971638083458, + "selected_utility_gain_over_base": -1.8951028361916542, + "selector_regret": 0.049669139087200165, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.012343091662249582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.2799132802532731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15051491558551788, + "selected_utility_gain_over_base": 0.07352641969919205, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": -0.06008357788835575, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14671264588832855, + "seed": "278008524/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.11082633957266808, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08110463619232178, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.05738186559193783, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": -0.007046685134192011, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.12610100641198746, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.0007701118593104184, + "selected_utility_gain_over_base": -1.9992298881406896, + "selector_regret": 1.9992298881406896, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07837157640798363, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.17583466760655425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "seed": "141984615/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0028510887641459703, + "selected_utility_gain_over_base": -0.09998315828852355, + "selector_regret": 0.012326816795393825, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.03405127834277031, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.6604019425937063, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.07178951263611572, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019201835617423058, + "selected_utility_gain_over_base": -0.034921107813715935, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.4228841070283557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.040833043884380325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.07837911942008637, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02024897001683712, + "selected_utility_gain_over_base": -0.07274852134287357, + "selector_regret": 0.021145423874258995, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2537693910556863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.39675023687824224, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06407292932271957, + "selected_utility_gain_over_base": -1.9359270706772804, + "selector_regret": 0.04432270675897598, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1994516379308126, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.07559139117061298, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014460672849353519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20374051418371542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.024846342330285976, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26309459020316783, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "seed": "1324642411/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0971866250038147, + "selected_utility_gain_over_base": -0.009283900260925293, + "selector_regret": 0.04687342047691345, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8949606216313842, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06870591640472412, + "selected_utility_gain_over_base": -0.056061550974845886, + "selector_regret": 1.9312940835952759, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.21147644486112652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.9127191083525623, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 14, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": -0.14699428323096494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.09862855674241659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.12276826591742303, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.133108988404274, + "selected_utility_gain_over_base": -0.029897689819335938, + "selector_regret": 0.0042196959257125854, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.08918541195471848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.07666470273917596, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 15, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.18909206180598703, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1175538864377494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1094239205121994, + "selected_utility_gain_over_base": -0.08148449659347534, + "selector_regret": 1.8905760794878006, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.15245882671444397, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.46521691519350344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.44073432486182534, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.17181242655217324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 13, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.20784347542059323, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.4539009809826104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.0797833736343915, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.042734309931976376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.22998146980120682, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.08946848235017621, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2553463076904834, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1630710940171585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09819497913122177, + "selected_utility_gain_over_base": -0.06349094957113266, + "selector_regret": 1.9018050208687782, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.2760414618475259, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0066074528731405735, + "selected_utility_gain_over_base": -0.08729561185464263, + "selector_regret": 0.00997756002470851, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.3453573992092333, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 9, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.009895960800349712, + "selected_utility_gain_over_base": 0.0079664591467008, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3074959621232845, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13291174173355103, + "selected_utility_gain_over_base": -1.867088258266449, + "selector_regret": 1.867088258266449, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.012343091662249582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.3657679062293992, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13429443538188934, + "selected_utility_gain_over_base": 0.05730593949556351, + "selector_regret": 0.07473599910736084, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": -0.0499732292936806, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03946288302540779, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0699788689083497, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": -0.003596889932491008, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.061759788542985916, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05349260475486517, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01598426327109337, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.23022103693086896, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 11, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.0732609664668121, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30974921814256706, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01966226100921631, + "selected_utility_gain_over_base": -0.08317198604345322, + "selector_regret": 0.017348866909742355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.03405127834277031, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.47366476193151535, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05904795601963997, + "selected_utility_gain_over_base": -0.0240870900452137, + "selector_regret": 0.008363228291273117, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.17917757553470504, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015374322421848774, + "selected_utility_gain_over_base": -0.03874862100929022, + "selector_regret": 0.02740760799497366, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.47260697010015135, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.040833043884380325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": -0.03458880944603605, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018512330949306488, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2537693910556863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.3230030628478873, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "seed": "747260437/train1", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06328952312469482, + "selected_utility_gain_over_base": -1.9367104768753052, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1994516379308126, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": -0.057844449538797256, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014460672849353519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20374051418371542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09844395714479388, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24354056683915523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09147362411022186, + "selected_utility_gain_over_base": -0.014996901154518127, + "selector_regret": 1.9085263758897781, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.9257790345269087, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10585354268550873, + "selected_utility_gain_over_base": -0.01891392469406128, + "selector_regret": 1.8941464573144913, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.04549911228110283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8886451280276686, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": -0.017059950476302946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.06662426715711083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1363699310338031, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "seed": "655884605/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12105989456176758, + "selected_utility_gain_over_base": -0.041946783661842346, + "selector_regret": 0.6918159127235413, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.06662426715711083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.054624554635065964, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.06863954075399922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 8, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14011503123535704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18476036190986633, + "selected_utility_gain_over_base": -0.006148055195808411, + "selector_regret": 0.02819681167602539, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.11782167529392211, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1151410346/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.47716841136356297, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6281729981841276, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8246148228645325, + "selected_utility_gain_over_base": -1.1753851771354675, + "selector_regret": 1.1753851771354675, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.05778003336251563, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.11203249181004805, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.38131013023777216, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.051708129348998336, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.042734309931976376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.2781639906385187, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "seed": "608016290/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.08946848235017621, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "seed": "441046872/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "task_id": "LiftPegUpright-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2553463076904834, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1630710940171585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.005216995719820261, + "selected_utility_gain_over_base": -0.15646893298253417, + "selector_regret": 1.9947830042801797, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.003297545598687419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.2249907445136055, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03885193541646004, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02248450182378292, + "selected_utility_gain_over_base": 0.020555000170134008, + "selector_regret": 0.016367433592677116, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21456877514719963, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07284616748064143, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05522008612751961, + "selected_utility_gain_over_base": -1.9447799138724804, + "selector_regret": 1.9447799138724804, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.012343091662249582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.23632015006049384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20389588177204132, + "selected_utility_gain_over_base": 0.12690738588571548, + "selector_regret": 0.014965444803237915, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": -0.0499732292936806, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 13, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0699788689083497, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 8, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": -0.0516450708448438, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.19208125940556758, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.002583444118499756, + "selected_utility_gain_over_base": -1.9974165558815002, + "selector_regret": 1.9974165558815002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07837157640798363, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.07434241180960115, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "seed": "141984615/train2", + "selected_candidate_index": 11, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.058951716870069504, + "selected_utility_gain_over_base": -0.04388253018260002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.0554029491396642, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12807528674602509, + "selected_utility_gain_over_base": -0.014633670449256897, + "selector_regret": 1.871924713253975, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.5311324826881051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02323226071894169, + "selected_utility_gain_over_base": -0.05990278534591198, + "selector_regret": 0.05241936258971691, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.11134770923200239, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04980705305933952, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035472434014081955, + "selected_utility_gain_over_base": -0.018650509417057037, + "selector_regret": 0.014334619045257568, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14730047062039375, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.47260697010015135, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.040833043884380325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.004969387149850623, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.051346004009246826, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1726434826850891, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2932448210537096, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.39675023687824224, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.03776244446635246, + "selected_utility_gain_over_base": -1.9622375555336475, + "selector_regret": 0.08733943477272987, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1994516379308126, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05637194588780403, + "selected_utility_gain_over_base": 0.00678827241063118, + "selector_regret": 1.943628054112196, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.07559139117061298, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "seed": "1549324805/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03407943248748779, + "selected_utility_gain_over_base": -0.07043540477752686, + "selector_regret": 0.07902240753173828, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014460672849353519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20374051418371542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.036805599015746454, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18492778771482454, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8433034939689589, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.04549911228110283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.0071199963429667, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": -0.097109495040569, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.08918541195471848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1353652692338349, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19850653409957886, + "selected_utility_gain_over_base": 0.03549985587596893, + "selector_regret": 0.01596677303314209, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.07658840863830661, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.054624554635065964, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 12, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.09945795364952359, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14011503123535704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "seed": "211245842/train2", + "selected_candidate_index": 10, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12799203395843506, + "selected_utility_gain_over_base": -0.06291638314723969, + "selector_regret": 0.10589073598384857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.11782167529392211, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "seed": "1151410346/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.46521691519350344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6798301258465532, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.05778003336251563, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1324114870513349, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.05424268514626182, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 7, + "random_state": 0 + }, + "selected_model_type": "hgb_regressor", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.4645161290322581, + "fallback_rate": 0.535483870967742, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.567741935483871, + "proposal_oracle_success": 0.5419354838709678, + "proposal_oracle_utility": 1.1626624377444386, + "selected_success": 0.38064516129032255, + "selected_success_gain_over_base": 0.1032258064516129, + "selected_utility": 0.8593619711727144, + "selector_regret": 0.36662355193928364, + "success_selector_gap": 0.1935483870967742, + "success_support_gap": 0.18064516129032257, + "support_gap": 0.3590716600478176 + }, + "selector_calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "selector_chart_feature_mode": "base_context_obs", + "selector_eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "selector_eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "selector_source_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "selector_source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "source_content_hash": null, + "source_index": null, + "source_split_hash": null, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5041666666666667, + "low": 0.3333333333333333, + "mean": 0.4222222222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4222222222222222, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 1.0, + "767747424/train1": 1.0, + "767747424/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5434343434343434, + "low": 0.05454545454545454, + "mean": 0.29898989898989903, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.29898989898989903, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.6666666666666666, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.5555555555555556, + "low": 0.3958333333333333, + "mean": 0.4791666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5041666666666667, + "low": 0.3333333333333333, + "mean": 0.4222222222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4222222222222222, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 1.0, + "767747424/train1": 1.0, + "767747424/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5434343434343434, + "low": 0.05454545454545454, + "mean": 0.29898989898989903, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.29898989898989903, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.6666666666666666, + "PushCube-v1": 0.5555555555555556, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.5555555555555556, + "low": 0.3958333333333333, + "mean": 0.4791666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6666666666666665, + "low": 0.49583333333333335, + "mean": 0.5777777777777777, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5777777777777777, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 1.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9454545454545455, + "low": 0.45656565656565656, + "mean": 0.7010101010101011, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7010101010101011, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.4444444444444444, + "StackCube-v1": 0.7272727272727273 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.6041666666666666, + "low": 0.4444444444444444, + "mean": 0.5208333333333334, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4430555555555556, + "mean": 0.5319444444444444, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5319444444444444, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7206060606060607, + "low": 0.2206060606060606, + "mean": 0.49393939393939396, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.49393939393939396, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.4722222222222222, + "mean": 0.5486111111111112, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.11729860108639417, + "low": 0.009807178363893774, + "mean": 0.06477040934233039, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.06477040934233039, + "per_seed": { + "1073161778/train0": -0.042734309931976376, + "1073161778/train1": -0.042734309931976376, + "1073161778/train2": -0.042734309931976376, + "1113371261/train0": -0.08918541195471848, + "1113371261/train1": -0.06662426715711083, + "1113371261/train2": -0.07658840863830661, + "1151410346/train0": -0.15245882671444397, + "1151410346/train1": -0.11782167529392211, + "1151410346/train2": -0.11782167529392211, + "1261930456/train0": -0.21147644486112652, + "1261930456/train1": -0.04549911228110283, + "1261930456/train2": -0.04549911228110283, + "1324133122/train0": -0.46521691519350344, + "1324133122/train1": -0.47716841136356297, + "1324133122/train2": -0.46521691519350344, + "1324642411/train0": 0.26309459020316783, + "1324642411/train1": 0.24354056683915523, + "1324642411/train2": 0.18492778771482454, + "1375492786/train0": -0.4228841070283557, + "1375492786/train1": -0.47260697010015135, + "1375492786/train2": -0.47260697010015135, + "141984615/train0": 0.17583466760655425, + "141984615/train1": 0.30974921814256706, + "141984615/train2": 0.07434241180960115, + "1461069713/train0": 0.003297545598687419, + "1461069713/train1": 0.2760414618475259, + "1461069713/train2": 0.003297545598687419, + "1480856810/train0": -0.14699428323096494, + "1480856810/train1": -0.017059950476302946, + "1480856810/train2": -0.097109495040569, + "1509980664/train0": -0.18909206180598703, + "1509980664/train1": -0.06863954075399922, + "1509980664/train2": -0.09945795364952359, + "1522272686/train0": -0.1172335482452192, + "1522272686/train1": -0.1172335482452192, + "1522272686/train2": -0.1172335482452192, + "1549324805/train0": 0.07559139117061298, + "1549324805/train1": -0.057844449538797256, + "1549324805/train2": 0.07559139117061298, + "1578990937/train0": -0.07666470273917596, + "1578990937/train1": -0.054624554635065964, + "1578990937/train2": -0.054624554635065964, + "1747178030/train0": 0.2553463076904834, + "1747178030/train1": 0.2553463076904834, + "1747178030/train2": 0.2553463076904834, + "1790188787/train0": 0.8949606216313842, + "1790188787/train1": 0.9257790345269087, + "1790188787/train2": 0.8433034939689589, + "1822210471/train0": 0.014460672849353519, + "1822210471/train1": 0.014460672849353519, + "1822210471/train2": 0.014460672849353519, + "1845489013/train0": 0.05738186559193783, + "1845489013/train1": 0.0699788689083497, + "1845489013/train2": 0.0699788689083497, + "1999022574/train0": -0.20784347542059323, + "1999022574/train1": -0.11203249181004805, + "1999022574/train2": -0.1324114870513349, + "2012211287/train0": -0.46177220122772705, + "2012211287/train1": -0.4539009809826104, + "2012211287/train2": -0.38131013023777216, + "2038594191/train0": 0.040833043884380325, + "2038594191/train1": 0.040833043884380325, + "2038594191/train2": 0.040833043884380325, + "2054899126/train0": 0.3402043551705839, + "2054899126/train1": 0.28063267658948543, + "2054899126/train2": 0.1830454942671755, + "2061635888/train0": -0.09862855674241659, + "2061635888/train1": -0.06662426715711083, + "2061635888/train2": -0.08918541195471848, + "211245842/train0": 0.1175538864377494, + "211245842/train1": 0.14011503123535704, + "211245842/train2": 0.14011503123535704, + "278008524/train0": 0.06968403002122843, + "278008524/train1": 0.07473920431856601, + "278008524/train2": 0.07473920431856601, + "307068879/train0": -0.084742699278881, + "307068879/train1": -0.0797833736343915, + "307068879/train2": -0.051708129348998336, + "412102563/train0": -0.17181242655217324, + "412102563/train1": -0.05778003336251563, + "412102563/train2": -0.05778003336251563, + "440577711/train0": 0.024846342330285976, + "440577711/train1": 0.09844395714479388, + "440577711/train2": 0.036805599015746454, + "441046872/train0": -0.034590442020628026, + "441046872/train1": -0.08946848235017621, + "441046872/train2": -0.08946848235017621, + "509430392/train0": 0.9127191083525623, + "509430392/train1": 0.8886451280276686, + "509430392/train2": 1.0071199963429667, + "585904130/train0": 0.07837157640798363, + "585904130/train1": 0.0732609664668121, + "585904130/train2": 0.07837157640798363, + "608016290/train0": -0.2053140291796788, + "608016290/train1": -0.22998146980120682, + "608016290/train2": -0.2781639906385187, + "626486813/train0": 0.16915535703752066, + "626486813/train1": 0.17088025463837114, + "626486813/train2": 0.08667283683438086, + "636680137/train0": 0.6604019425937063, + "636680137/train1": 0.47366476193151535, + "636680137/train2": 0.5311324826881051, + "655884605/train0": 0.12276826591742303, + "655884605/train1": 0.1363699310338031, + "655884605/train2": 0.1353652692338349, + "672328262/train0": -0.012343091662249582, + "672328262/train1": -0.012343091662249582, + "672328262/train2": -0.012343091662249582, + "701667486/train0": 0.44073432486182534, + "701667486/train1": 0.6281729981841276, + "701667486/train2": 0.6798301258465532, + "747260437/train0": 0.39675023687824224, + "747260437/train1": 0.3230030628478873, + "747260437/train2": 0.39675023687824224, + "750448296/train0": 0.1053903039821907, + "750448296/train1": 0.06773432769348323, + "750448296/train2": 0.08803761682441008, + "767747424/train0": 0.16336797415873666, + "767747424/train1": 0.23455940317803292, + "767747424/train2": 0.1757050531788056 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.20520628335335273, + "low": -0.11515179902113089, + "mean": 0.025837563222809418, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.025837563222809418, + "per_task": { + "LiftPegUpright-v1": -0.17122715605644628, + "PickCube-v1": -0.1172335482452192, + "PullCube-v1": 0.12865684899902294, + "PushCube-v1": 0.32003043488484745, + "StackCube-v1": -0.03103876346815781 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.12270200376483784, + "low": 0.036902445425064316, + "mean": 0.07962100224182234, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6722222222222223, + "low": 0.5055555555555555, + "mean": 0.5847222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5847222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7430303030303029, + "low": 0.22303030303030305, + "mean": 0.5074747474747474, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5074747474747474, + "per_task": { + "LiftPegUpright-v1": 0.6, + "PickCube-v1": 0.0, + "PullCube-v1": 0.5333333333333333, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.5151515151515151 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.6458333333333334, + "low": 0.4930555555555556, + "mean": 0.5694444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4116970675635254, + "low": 1.1051828979592149, + "mean": 1.251578581963097, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.251578581963097, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 2.0, + "1151410346/train2": 0.5522047281265259, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.14406004548072815, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.015177905559539795, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09997648000717163, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.060871422290802, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.11310184001922607, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.23388276994228363, + "278008524/train0": 1.0733563229441643, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.2302134484052658, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7191348671913147, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.7117276191711426, + "626486813/train0": 0.04469547560438514, + "626486813/train1": 0.035827874671667814, + "626486813/train2": 0.04489925503730774, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.8128758072853088, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.3504486680030823, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 1.0629530912265182, + "767747424/train2": 1.0671670949086547 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5246927151765788, + "low": 0.5848668839583795, + "mean": 1.1196794969856438, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.1196794969856438, + "per_task": { + "LiftPegUpright-v1": 1.4122399787108104, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 1.1093241602430741, + "PushCube-v1": 1.7857814149724112, + "StackCube-v1": 1.1567951707225856 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.360821089959548, + "low": 1.0729441113071516, + "mean": 1.2159999172436073, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 4.099305555555556, + "low": 2.800694444444445, + "mean": 3.4395833333333337, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.4395833333333337, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 2.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 0.0, + "1151410346/train1": 2.0, + "1151410346/train2": 1.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 4.0, + "1324133122/train0": 6.0, + "1324133122/train1": 5.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 3.0, + "1375492786/train0": 0.0, + "1375492786/train1": 2.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 11.0, + "1461069713/train0": 2.0, + "1461069713/train1": 1.0, + "1461069713/train2": 5.0, + "1480856810/train0": 0.0, + "1480856810/train1": 8.0, + "1480856810/train2": 1.0, + "1509980664/train0": 13.0, + "1509980664/train1": 8.0, + "1509980664/train2": 5.0, + "1522272686/train0": 1.0, + "1522272686/train1": 2.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 15.0, + "1578990937/train1": 3.0, + "1578990937/train2": 12.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 7.0, + "1822210471/train1": 5.0, + "1822210471/train2": 0.0, + "1845489013/train0": 5.0, + "1845489013/train1": 7.0, + "1845489013/train2": 8.0, + "1999022574/train0": 5.0, + "1999022574/train1": 8.0, + "1999022574/train2": 6.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.5, + "2061635888/train0": 3.0, + "2061635888/train1": 2.0, + "2061635888/train2": 7.0, + "211245842/train0": 4.0, + "211245842/train1": 2.0, + "211245842/train2": 10.0, + "278008524/train0": 6.0, + "278008524/train1": 5.5, + "278008524/train2": 11.5, + "307068879/train0": 1.0, + "307068879/train1": 6.0, + "307068879/train2": 0.0, + "412102563/train0": 13.0, + "412102563/train1": 8.0, + "412102563/train2": 4.0, + "440577711/train0": 3.0, + "440577711/train1": 5.0, + "440577711/train2": 2.0, + "441046872/train0": 1.0, + "441046872/train1": 5.0, + "441046872/train2": 3.0, + "509430392/train0": 14.0, + "509430392/train1": 2.0, + "509430392/train2": 1.0, + "585904130/train0": 4.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 8.0, + "608016290/train1": 11.0, + "608016290/train2": 13.0, + "626486813/train0": 3.0, + "626486813/train1": 4.5, + "626486813/train2": 3.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 3.0, + "655884605/train1": 8.0, + "655884605/train2": 3.0, + "672328262/train0": 8.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 7.0, + "701667486/train1": 7.0, + "701667486/train2": 5.0, + "747260437/train0": 0.0, + "747260437/train1": 10.0, + "747260437/train2": 0.0, + "750448296/train0": 2.6666666666666665, + "750448296/train1": 1.6666666666666667, + "750448296/train2": 1.6666666666666667, + "767747424/train0": 3.25, + "767747424/train1": 3.75, + "767747424/train2": 1.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 4.3447272727272725, + "low": 2.0186666666666664, + "mean": 3.109575757575757, + "n": 5, + "num_samples": 1000 + }, + "mean": 3.109575757575757, + "per_task": { + "LiftPegUpright-v1": 3.4, + "PickCube-v1": 1.3333333333333333, + "PullCube-v1": 2.6933333333333334, + "PushCube-v1": 2.6666666666666665, + "StackCube-v1": 5.454545454545454 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.9930555555555554, + "low": 2.7708333333333335, + "mean": 3.3680555555555554, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4034722222222223, + "low": 0.23680555555555552, + "mean": 0.32291666666666674, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.32291666666666674, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3511111111111111, + "low": 0.10933333333333332, + "mean": 0.23559595959595958, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.23559595959595958, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3466666666666667, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.24242424242424243 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3819444444444444, + "low": 0.22916666666666666, + "mean": 0.3055555555555556, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.07708333333333334, + "low": -0.03958333333333333, + "mean": 0.016666666666666666, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.016666666666666666, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": -1.0, + "701667486/train2": 0.0, + "747260437/train0": -1.0, + "747260437/train1": -1.0, + "747260437/train2": -1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": -0.25, + "767747424/train1": 0.0, + "767747424/train2": -0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.13066666666666665, + "low": -0.020121212121212123, + "mean": 0.03571717171717172, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.03571717171717172, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.013333333333333334, + "PushCube-v1": 0.2222222222222222, + "StackCube-v1": -0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.06944444444444445, + "low": -0.04861111111111111, + "mean": 0.013888888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9666446016434848, + "low": 0.6676490235895471, + "mean": 0.818970424955761, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.818970424955761, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.0971866250038147, + "1324642411/train1": 0.09147362411022186, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.0028510887641459703, + "141984615/train1": 0.01966226100921631, + "141984615/train2": 0.058951716870069504, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0066074528731405735, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.03407943248748779, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.06870591640472412, + "1790188787/train1": 0.10585354268550873, + "1790188787/train2": 2.0, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.052448581904173, + "2054899126/train1": 1.0664558708667755, + "2054899126/train2": 1.0276100430637598, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.1094239205121994, + "211245842/train1": 0.18476036190986633, + "211245842/train2": 0.12799203395843506, + "278008524/train0": 1.0179431531578302, + "278008524/train1": 1.0179431531578302, + "278008524/train2": 0.046129126101732254, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 2.0, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 0.12097799777984619, + "509430392/train1": 2.0, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 0.09919407218694687, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.007704744581133127, + "626486813/train1": 0.00908157229423523, + "626486813/train2": 0.015375842805951834, + "636680137/train0": 0.056092143058776855, + "636680137/train1": 0.05904795601963997, + "636680137/train2": 0.02323226071894169, + "655884605/train0": 0.133108988404274, + "655884605/train1": 0.12105989456176758, + "655884605/train2": 0.19850653409957886, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 0.8246148228645325, + "701667486/train2": 2.0, + "747260437/train0": 0.06407292932271957, + "747260437/train1": 0.06328952312469482, + "747260437/train2": 0.03776244446635246, + "750448296/train0": 0.720985975737373, + "750448296/train1": 0.7452354828516642, + "750448296/train2": 0.7403575927019119, + "767747424/train0": 0.050133669123169966, + "767747424/train1": 0.56196593423374, + "767747424/train2": 0.06179218890611082 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8639677355867444, + "low": 0.6614040293260477, + "mean": 0.7648172021960191, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7648172021960191, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.731942518534294, + "PushCube-v1": 0.8232003202040991, + "StackCube-v1": 0.7351135308995391 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.9064999793675573, + "low": 0.6199989132203174, + "mean": 0.7636942816805257, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14168937444321475, + "low": -0.07295669507750265, + "mean": 0.028852362657966698, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.028852362657966698, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -0.009283900260925293, + "1324642411/train1": -0.014996901154518127, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": -0.09998315828852355, + "141984615/train1": -0.08317198604345322, + "141984615/train2": -0.04388253018260002, + "1461069713/train0": 0.0, + "1461069713/train1": -0.08729561185464263, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": 0.0, + "1549324805/train2": -0.07043540477752686, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": -0.056061550974845886, + "1790188787/train1": -0.01891392469406128, + "1790188787/train2": 1.87523253262043, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9475514180958271, + "2054899126/train1": -0.9335441291332245, + "2054899126/train2": -0.9723899569362402, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": -0.08148449659347534, + "211245842/train1": -0.006148055195808411, + "211245842/train2": -0.06291638314723969, + "278008524/train0": 0.9752081632614136, + "278008524/train1": 0.9752081632614136, + "278008524/train2": 0.00339413620531559, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.12097777426242828, + "509430392/train1": 1.999999776482582, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": -0.0825214758515358, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0026064018602482975, + "626486813/train1": 0.0039832295733504, + "626486813/train2": 0.010277500085067004, + "636680137/train0": -0.027042903006076813, + "636680137/train1": -0.0240870900452137, + "636680137/train2": -0.05990278534591198, + "655884605/train0": -0.029897689819335938, + "655884605/train1": -0.041946783661842346, + "655884605/train2": 0.03549985587596893, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": -1.1753851771354675, + "701667486/train2": 0.0, + "747260437/train0": -1.9359270706772804, + "747260437/train1": -1.9367104768753052, + "747260437/train2": -1.9622375555336475, + "750448296/train0": 0.6033928263932467, + "750448296/train1": 0.6276423335075378, + "750448296/train2": 0.6227644433577856, + "767747424/train0": -0.5230656728817848, + "767747424/train1": -0.011233407771214843, + "767747424/train2": -0.511407153098844 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2502214501301448, + "low": -0.03419300033881639, + "mean": 0.069257405022004, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.069257405022004, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.043227942689166715, + "PushCube-v1": 0.43079614142576855, + "StackCube-v1": -0.041281173626581825 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.13145425815067333, + "low": -0.09352178430304371, + "mean": 0.0218746952381884, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5840604509834926, + "low": 0.3291477927744482, + "mean": 0.4515619357262443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4515619357262443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 1.7739987075328827, + "1151410346/train2": 0.32620343565940857, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.04687342047691345, + "1324642411/train1": 1.9085263758897781, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.012326816795393825, + "141984615/train1": 0.017348866909742355, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.00997756002470851, + "1461069713/train2": 0.006073415279388428, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.9129659235477448, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 1.1665899753570557, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0347811970859766, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.07902240753173828, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.9312940835952759, + "1790188787/train1": 1.8941464573144913, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.024834569543600082, + "2054899126/train1": 0.9335441291332245, + "2054899126/train2": 0.9723899569362402, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8905760794878006, + "211245842/train1": 0.02819681167602539, + "211245842/train2": 0.10589073598384857, + "278008524/train0": 0.05541316978633404, + "278008524/train1": 0.019731441512703896, + "278008524/train2": 1.08981978520751, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 0.0, + "440577711/train2": 1.9638472199440002, + "441046872/train0": 1.3000448346138, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.019179701805114746, + "509430392/train0": 1.8790220022201538, + "509430392/train1": 0.0, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 1.9008059278130531, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.03699073102325201, + "626486813/train1": 0.026746302377432585, + "626486813/train2": 0.029523412231355906, + "636680137/train0": 0.0321691632270813, + "636680137/train1": 0.008363228291273117, + "636680137/train2": 0.05241936258971691, + "655884605/train0": 0.0042196959257125854, + "655884605/train1": 0.6918159127235413, + "655884605/train2": 0.01596677303314209, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.1753851771354675, + "701667486/train2": 0.0, + "747260437/train0": 0.04432270675897598, + "747260437/train1": 0.0, + "747260437/train2": 0.08733943477272987, + "750448296/train0": 0.03167435216406981, + "750448296/train1": 0.6252677912513415, + "750448296/train2": 0.6239749044179916, + "767747424/train0": 0.9922955186775653, + "767747424/train1": 0.5009871569927782, + "767747424/train2": 1.0053749060025439 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7381252358189615, + "low": 0.17861606451729312, + "mean": 0.4560315842593437, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4560315842593437, + "per_task": { + "LiftPegUpright-v1": 0.5146939317385356, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3783863908479301, + "PushCube-v1": 0.9625810947683122, + "StackCube-v1": 0.42449650394194055 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.5983463580620688, + "low": 0.3538582401920899, + "mean": 0.4682932821221281, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.34236111111111106, + "low": 0.18888888888888886, + "mean": 0.2618055555555555, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.2618055555555555, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40909090909090906, + "low": 0.11200000000000002, + "mean": 0.2718787878787879, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2718787878787879, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.18666666666666668, + "PushCube-v1": 0.5, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3402777777777778, + "low": 0.19444444444444445, + "mean": 0.2638888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.30416666666666664, + "low": 0.15833333333333333, + "mean": 0.23194444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.23194444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7191919191919192, + "low": 0.13511111111111113, + "mean": 0.40452525252525257, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.40452525252525257, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 1.0, + "PullCube-v1": 0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.48484848484848486 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.2638888888888889, + "low": 0.1388888888888889, + "mean": 0.2013888888888889, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5434736852068454, + "low": 0.2965679521788843, + "mean": 0.4232947583840643, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4232947583840643, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 0.0, + "1151410346/train2": 1.4477952718734741, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8559399545192719, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.270599365234375, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2268255539238453, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.05936597287654877, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.23658466339111328, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.1843542456626892, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7661172300577164, + "278008524/train0": 0.04055231809616089, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7697865515947342, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.2808651328086853, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.2882723808288574, + "626486813/train0": 0.12161866808310151, + "626486813/train1": 0.12975450651720166, + "626486813/train2": 0.12068312615156174, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.1871241927146912, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.21537372469902039, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05754782756169637, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.045098946429789066, + "767747424/train2": 0.04088494274765253 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.330930626693398, + "low": 0.24626440485649642, + "mean": 0.7236978183567826, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7236978183567826, + "per_task": { + "LiftPegUpright-v1": 0.5877600212891897, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.1075624164690574, + "PushCube-v1": 0.21421858502758873, + "StackCube-v1": 0.8432048292774142 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.485598717091812, + "low": 0.272009730946997, + "mean": 0.3761285081272945, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.05424268514626184, + "low": 0.05424268514626184, + "mean": 0.05424268514626184, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.05424268514626184, + "per_seed": { + "1073161778/train0": 0.05424268514626182, + "1073161778/train1": 0.05424268514626182, + "1073161778/train2": 0.05424268514626182, + "1113371261/train0": 0.05424268514626182, + "1113371261/train1": 0.05424268514626182, + "1113371261/train2": 0.05424268514626182, + "1151410346/train0": 0.05424268514626182, + "1151410346/train1": 0.05424268514626182, + "1151410346/train2": 0.05424268514626182, + "1261930456/train0": 0.05424268514626182, + "1261930456/train1": 0.05424268514626182, + "1261930456/train2": 0.05424268514626182, + "1324133122/train0": 0.05424268514626182, + "1324133122/train1": 0.05424268514626182, + "1324133122/train2": 0.05424268514626182, + "1324642411/train0": 0.05424268514626182, + "1324642411/train1": 0.05424268514626182, + "1324642411/train2": 0.05424268514626182, + "1375492786/train0": 0.05424268514626182, + "1375492786/train1": 0.05424268514626182, + "1375492786/train2": 0.05424268514626182, + "141984615/train0": 0.05424268514626182, + "141984615/train1": 0.05424268514626182, + "141984615/train2": 0.05424268514626182, + "1461069713/train0": 0.05424268514626182, + "1461069713/train1": 0.05424268514626182, + "1461069713/train2": 0.05424268514626182, + "1480856810/train0": 0.05424268514626182, + "1480856810/train1": 0.05424268514626182, + "1480856810/train2": 0.05424268514626182, + "1509980664/train0": 0.05424268514626182, + "1509980664/train1": 0.05424268514626182, + "1509980664/train2": 0.05424268514626182, + "1522272686/train0": 0.05424268514626182, + "1522272686/train1": 0.05424268514626182, + "1522272686/train2": 0.05424268514626182, + "1549324805/train0": 0.05424268514626182, + "1549324805/train1": 0.05424268514626182, + "1549324805/train2": 0.05424268514626182, + "1578990937/train0": 0.05424268514626182, + "1578990937/train1": 0.05424268514626182, + "1578990937/train2": 0.05424268514626182, + "1747178030/train0": 0.05424268514626182, + "1747178030/train1": 0.05424268514626182, + "1747178030/train2": 0.05424268514626182, + "1790188787/train0": 0.05424268514626182, + "1790188787/train1": 0.05424268514626182, + "1790188787/train2": 0.05424268514626182, + "1822210471/train0": 0.05424268514626182, + "1822210471/train1": 0.05424268514626182, + "1822210471/train2": 0.05424268514626182, + "1845489013/train0": 0.05424268514626182, + "1845489013/train1": 0.05424268514626182, + "1845489013/train2": 0.05424268514626182, + "1999022574/train0": 0.05424268514626182, + "1999022574/train1": 0.05424268514626182, + "1999022574/train2": 0.05424268514626182, + "2012211287/train0": 0.05424268514626182, + "2012211287/train1": 0.05424268514626182, + "2012211287/train2": 0.05424268514626182, + "2038594191/train0": 0.05424268514626182, + "2038594191/train1": 0.05424268514626182, + "2038594191/train2": 0.05424268514626182, + "2054899126/train0": 0.05424268514626182, + "2054899126/train1": 0.05424268514626182, + "2054899126/train2": 0.05424268514626182, + "2061635888/train0": 0.05424268514626182, + "2061635888/train1": 0.05424268514626182, + "2061635888/train2": 0.05424268514626182, + "211245842/train0": 0.05424268514626182, + "211245842/train1": 0.05424268514626182, + "211245842/train2": 0.05424268514626182, + "278008524/train0": 0.05424268514626182, + "278008524/train1": 0.05424268514626182, + "278008524/train2": 0.05424268514626182, + "307068879/train0": 0.05424268514626182, + "307068879/train1": 0.05424268514626182, + "307068879/train2": 0.05424268514626182, + "412102563/train0": 0.05424268514626182, + "412102563/train1": 0.05424268514626182, + "412102563/train2": 0.05424268514626182, + "440577711/train0": 0.05424268514626182, + "440577711/train1": 0.05424268514626182, + "440577711/train2": 0.05424268514626182, + "441046872/train0": 0.05424268514626182, + "441046872/train1": 0.05424268514626182, + "441046872/train2": 0.05424268514626182, + "509430392/train0": 0.05424268514626182, + "509430392/train1": 0.05424268514626182, + "509430392/train2": 0.05424268514626182, + "585904130/train0": 0.05424268514626182, + "585904130/train1": 0.05424268514626182, + "585904130/train2": 0.05424268514626182, + "608016290/train0": 0.05424268514626182, + "608016290/train1": 0.05424268514626182, + "608016290/train2": 0.05424268514626182, + "626486813/train0": 0.05424268514626182, + "626486813/train1": 0.05424268514626182, + "626486813/train2": 0.05424268514626182, + "636680137/train0": 0.05424268514626182, + "636680137/train1": 0.05424268514626182, + "636680137/train2": 0.05424268514626182, + "655884605/train0": 0.05424268514626182, + "655884605/train1": 0.05424268514626182, + "655884605/train2": 0.05424268514626182, + "672328262/train0": 0.05424268514626182, + "672328262/train1": 0.05424268514626182, + "672328262/train2": 0.05424268514626182, + "701667486/train0": 0.05424268514626182, + "701667486/train1": 0.05424268514626182, + "701667486/train2": 0.05424268514626182, + "747260437/train0": 0.05424268514626182, + "747260437/train1": 0.05424268514626182, + "747260437/train2": 0.05424268514626182, + "750448296/train0": 0.05424268514626182, + "750448296/train1": 0.05424268514626182, + "750448296/train2": 0.05424268514626182, + "767747424/train0": 0.05424268514626182, + "767747424/train1": 0.05424268514626182, + "767747424/train2": 0.05424268514626182 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.054242685146261835, + "low": 0.05424268514626183, + "mean": 0.054242685146261835, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.054242685146261835, + "per_task": { + "LiftPegUpright-v1": 0.05424268514626184, + "PickCube-v1": 0.05424268514626182, + "PullCube-v1": 0.054242685146261835, + "PushCube-v1": 0.05424268514626184, + "StackCube-v1": 0.05424268514626184 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.05424268514626184, + "low": 0.05424268514626184, + "mean": 0.05424268514626184, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "utility_margin", + "tau": 0.05424268514626182 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..c5a8f903dd9af1a75c0f0f7b20d78c4f9f2354c2 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.042734309931976376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.042734309931976376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.042734309931976376, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.08918541195471848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.05424268514626182 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.06662426715711083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.05424268514626182 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.07658840863830661, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.05424268514626182 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.15245882671444397, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.05424268514626182 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.11782167529392211, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.7739987075328827, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.11782167529392211, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5522047281265259, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.32620343565940857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4477952718734741, + "tau": 0.05424268514626182 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.21147644486112652, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.04549911228110283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.04549911228110283, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.46521691519350344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.05424268514626182 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.47716841136356297, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.05424268514626182 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.46521691519350344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.05424268514626182 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26309459020316783, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14406004548072815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0971866250038147, + "selected_utility_gain_over_base": -0.009283900260925293, + "selector_regret": 0.04687342047691345, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8559399545192719, + "tau": 0.05424268514626182 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24354056683915523, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09147362411022186, + "selected_utility_gain_over_base": -0.014996901154518127, + "selector_regret": 1.9085263758897781, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18492778771482454, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.89352947473526, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.4228841070283557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.47260697010015135, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.47260697010015135, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.17583466760655425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.015177905559539795, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0028510887641459703, + "selected_utility_gain_over_base": -0.09998315828852355, + "selector_regret": 0.012326816795393825, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.270599365234375, + "tau": 0.05424268514626182 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.30974921814256706, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01966226100921631, + "selected_utility_gain_over_base": -0.08317198604345322, + "selector_regret": 0.017348866909742355, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.05424268514626182 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.07434241180960115, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.058951716870069504, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.058951716870069504, + "selected_utility_gain_over_base": -0.04388253018260002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2268255539238453, + "tau": 0.05424268514626182 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.003297545598687419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.05424268514626182 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.2760414618475259, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0066074528731405735, + "selected_utility_gain_over_base": -0.08729561185464263, + "selector_regret": 0.00997756002470851, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.05424268514626182 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.003297545598687419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09997648000717163, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006073415279388428, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.05936597287654877, + "tau": 0.05424268514626182 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": -0.14699428323096494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": -0.017059950476302946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": -0.097109495040569, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.18909206180598703, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.06863954075399922, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.09945795364952359, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.05424268514626182 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.05424268514626182 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.05424268514626182 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.07559139117061298, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.060871422290802, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.0347811970859766, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.23658466339111328, + "tau": 0.05424268514626182 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": -0.057844449538797256, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.05424268514626182 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.07559139117061298, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11310184001922607, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03407943248748779, + "selected_utility_gain_over_base": -0.07043540477752686, + "selector_regret": 0.07902240753173828, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1843542456626892, + "tau": 0.05424268514626182 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.07666470273917596, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 15.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.054624554635065964, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.054624554635065964, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 12.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2553463076904834, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2553463076904834, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2553463076904834, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8949606216313842, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06870591640472412, + "selected_utility_gain_over_base": -0.056061550974845886, + "selector_regret": 1.9312940835952759, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.9257790345269087, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10585354268550873, + "selected_utility_gain_over_base": -0.01891392469406128, + "selector_regret": 1.8941464573144913, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8433034939689589, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014460672849353519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014460672849353519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014460672849353519, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.05738186559193783, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0699788689083497, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0699788689083497, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.20784347542059323, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.11203249181004805, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1324114870513349, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.46177220122772705, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.4539009809826104, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.38131013023777216, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.040833043884380325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.040833043884380325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.040833043884380325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3402043551705839, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.052448581904173, + "selected_utility_gain_over_base": -0.9475514180958271, + "selector_regret": 0.024834569543600082, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.05424268514626182 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28063267658948543, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0664558708667755, + "selected_utility_gain_over_base": -0.9335441291332245, + "selector_regret": 0.9335441291332245, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1830454942671755, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0276100430637598, + "selected_utility_gain_over_base": -0.9723899569362402, + "selector_regret": 0.9723899569362402, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.09862855674241659, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.05424268514626182 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.06662426715711083, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.05424268514626182 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.08918541195471848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.05424268514626182 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1175538864377494, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1094239205121994, + "selected_utility_gain_over_base": -0.08148449659347534, + "selector_regret": 1.8905760794878006, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14011503123535704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18476036190986633, + "selected_utility_gain_over_base": -0.006148055195808411, + "selector_regret": 0.02819681167602539, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.05424268514626182 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14011503123535704, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.23388276994228363, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12799203395843506, + "selected_utility_gain_over_base": -0.06291638314723969, + "selector_regret": 0.10589073598384857, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7661172300577164, + "tau": 0.05424268514626182 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.06968403002122843, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0733563229441643, + "selected_candidate_index": 6.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0179431531578302, + "selected_utility_gain_over_base": 0.9752081632614136, + "selector_regret": 0.05541316978633404, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.04055231809616089, + "tau": 0.05424268514626182 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.07473920431856601, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 5.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0179431531578302, + "selected_utility_gain_over_base": 0.9752081632614136, + "selector_regret": 0.019731441512703896, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.05424268514626182 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.07473920431856601, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 11.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.046129126101732254, + "selected_utility_gain_over_base": 0.00339413620531559, + "selector_regret": 1.08981978520751, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.084742699278881, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.05424268514626182 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.0797833736343915, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.05424268514626182 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.051708129348998336, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2302134484052658, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7697865515947342, + "tau": 0.05424268514626182 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.17181242655217324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 13.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.05778003336251563, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.05778003336251563, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.024846342330285976, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09844395714479388, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.036805599015746454, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.034590442020628026, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.3000448346138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.08946848235017621, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.05424268514626182 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.08946848235017621, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7191348671913147, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.019179701805114746, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2808651328086853, + "tau": 0.05424268514626182 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.9127191083525623, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 14.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12097799777984619, + "selected_utility_gain_over_base": 0.12097777426242828, + "selector_regret": 1.8790220022201538, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8886451280276686, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.999999776482582, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 1.0071199963429667, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.05424268514626182 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07837157640798363, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09919407218694687, + "selected_utility_gain_over_base": -0.0825214758515358, + "selector_regret": 1.9008059278130531, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.0732609664668121, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.07837157640798363, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.2053140291796788, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": -0.22998146980120682, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 11.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.2781639906385187, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7117276191711426, + "selected_candidate_index": 13.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2882723808288574, + "tau": 0.05424268514626182 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.16915535703752066, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04469547560438514, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.007704744581133127, + "selected_utility_gain_over_base": 0.0026064018602482975, + "selector_regret": 0.03699073102325201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12161866808310151, + "tau": 0.05424268514626182 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.17088025463837114, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.035827874671667814, + "selected_candidate_index": 4.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00908157229423523, + "selected_utility_gain_over_base": 0.0039832295733504, + "selector_regret": 0.026746302377432585, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12975450651720166, + "tau": 0.05424268514626182 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.08667283683438086, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04489925503730774, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015375842805951834, + "selected_utility_gain_over_base": 0.010277500085067004, + "selector_regret": 0.029523412231355906, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12068312615156174, + "tau": 0.05424268514626182 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.6604019425937063, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.056092143058776855, + "selected_utility_gain_over_base": -0.027042903006076813, + "selector_regret": 0.0321691632270813, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.05424268514626182 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.47366476193151535, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05904795601963997, + "selected_utility_gain_over_base": -0.0240870900452137, + "selector_regret": 0.008363228291273117, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.05424268514626182 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.5311324826881051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02323226071894169, + "selected_utility_gain_over_base": -0.05990278534591198, + "selector_regret": 0.05241936258971691, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.05424268514626182 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.12276826591742303, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.133108988404274, + "selected_utility_gain_over_base": -0.029897689819335938, + "selector_regret": 0.0042196959257125854, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.05424268514626182 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1363699310338031, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8128758072853088, + "selected_candidate_index": 8.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12105989456176758, + "selected_utility_gain_over_base": -0.041946783661842346, + "selector_regret": 0.6918159127235413, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1871241927146912, + "tau": 0.05424268514626182 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1353652692338349, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19850653409957886, + "selected_utility_gain_over_base": 0.03549985587596893, + "selector_regret": 0.01596677303314209, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.05424268514626182 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.012343091662249582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 8.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.012343091662249582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.012343091662249582, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.44073432486182534, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6281729981841276, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8246148228645325, + "selected_utility_gain_over_base": -1.1753851771354675, + "selector_regret": 1.1753851771354675, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6798301258465532, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.05424268514626182 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.39675023687824224, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06407292932271957, + "selected_utility_gain_over_base": -1.9359270706772804, + "selector_regret": 0.04432270675897598, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.05424268514626182 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.3230030628478873, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06328952312469482, + "selected_candidate_index": 10.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06328952312469482, + "selected_utility_gain_over_base": -1.9367104768753052, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.21537372469902039, + "tau": 0.05424268514626182 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.39675023687824224, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.03776244446635246, + "selected_utility_gain_over_base": -1.9622375555336475, + "selector_regret": 0.08733943477272987, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.05424268514626182 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.1053903039821907, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 2.6666666666666665, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.720985975737373, + "selected_utility_gain_over_base": 0.6033928263932467, + "selector_regret": 0.03167435216406981, + "success_selector_gap": 0.0, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.05424268514626182 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.06773432769348323, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 1.6666666666666667, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.7452354828516642, + "selected_utility_gain_over_base": 0.6276423335075378, + "selector_regret": 0.6252677912513415, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.05424268514626182 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.08803761682441008, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3504486680030823, + "selected_candidate_index": 1.6666666666666667, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.7403575927019119, + "selected_utility_gain_over_base": 0.6227644433577856, + "selector_regret": 0.6239749044179916, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.0, + "support_gap": 0.05754782756169637, + "tau": 0.05424268514626182 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.16336797415873666, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 3.25, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.050133669123169966, + "selected_utility_gain_over_base": -0.5230656728817848, + "selector_regret": 0.9922955186775653, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.05424268514626182 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.23455940317803292, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0629530912265182, + "selected_candidate_index": 3.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.56196593423374, + "selected_utility_gain_over_base": -0.011233407771214843, + "selector_regret": 0.5009871569927782, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.045098946429789066, + "tau": 0.05424268514626182 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.1757050531788056, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0671670949086547, + "selected_candidate_index": 1.75, + "selected_success": 0.0, + "selected_success_gain_over_base": -0.25, + "selected_utility": 0.06179218890611082, + "selected_utility_gain_over_base": -0.511407153098844, + "selector_regret": 1.0053749060025439, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.04088494274765253, + "tau": 0.05424268514626182 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..2bb8f191988d2ea4e36b18c37ce128653823c28b --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5333333333333333, + "predicted_margin": -0.17122715605644628, + "proposal_oracle_success": 0.6, + "proposal_oracle_utility": 1.4122399787108104, + "selected_candidate_index": 3.4, + "selected_success": 0.2, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.9498705387115478, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5146939317385356, + "success_selector_gap": 0.4, + "success_support_gap": 0.4, + "support_gap": 0.5877600212891897, + "tau": 0.05424268514626184 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": -0.1172335482452192, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 1.3333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.05424268514626182 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.12865684899902294, + "proposal_oracle_success": 0.5333333333333333, + "proposal_oracle_utility": 1.1093241602430741, + "selected_candidate_index": 2.6933333333333334, + "selected_success": 0.3466666666666667, + "selected_success_gain_over_base": -0.013333333333333334, + "selected_utility": 0.731942518534294, + "selected_utility_gain_over_base": -0.043227942689166715, + "selector_regret": 0.3783863908479301, + "success_selector_gap": 0.18666666666666668, + "success_support_gap": 0.02666666666666667, + "support_gap": 0.1075624164690574, + "tau": 0.054242685146261835 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.5555555555555556, + "execute_generated": 0.5555555555555556, + "fallback_rate": 0.4444444444444444, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.8333333333333334, + "predicted_margin": 0.32003043488484745, + "proposal_oracle_success": 0.8888888888888888, + "proposal_oracle_utility": 1.7857814149724112, + "selected_candidate_index": 2.6666666666666665, + "selected_success": 0.3888888888888889, + "selected_success_gain_over_base": 0.2222222222222222, + "selected_utility": 0.8232003202040991, + "selected_utility_gain_over_base": 0.43079614142576855, + "selector_regret": 0.9625810947683122, + "success_selector_gap": 0.5, + "success_support_gap": 0.1111111111111111, + "support_gap": 0.21421858502758873, + "tau": 0.05424268514626184 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.2727272727272727, + "execute_generated": 0.2727272727272727, + "fallback_rate": 0.7272727272727273, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.6363636363636364, + "predicted_margin": -0.03103876346815781, + "proposal_oracle_success": 0.5151515151515151, + "proposal_oracle_utility": 1.1567951707225856, + "selected_candidate_index": 5.454545454545454, + "selected_success": 0.24242424242424243, + "selected_success_gain_over_base": -0.030303030303030304, + "selected_utility": 0.7351135308995391, + "selected_utility_gain_over_base": -0.041281173626581825, + "selector_regret": 0.42449650394194055, + "success_selector_gap": 0.2727272727272727, + "success_support_gap": 0.48484848484848486, + "support_gap": 0.8432048292774142, + "tau": 0.05424268514626184 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..cb9e0f5456c9539b99277ec507009d720a7dc7c1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_regressor` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 7, "random_state": 0}` +Tau: `0.054243` +Feature set: `chart_compat` +Target: `utility_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4224 | 0.5776 | 0.2780 | 0.4332 | 0.5343 | 0.5343 | 0.1733 | 0.1552 | +| selection | 0.4645 | 0.5355 | 0.2774 | 0.3806 | 0.5419 | 0.5677 | 0.1806 | 0.1935 | +| eval | 0.4792 | 0.5208 | 0.2917 | 0.3056 | 0.5694 | 0.5486 | 0.2014 | 0.2639 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..2706c3e0547971eab692012771ffa1c44982d2e5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/split_hash.txt @@ -0,0 +1,20 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + }, + "source_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..e55f5fb9d0fe4d2ed1b3dcc66129290a8fd09d64 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.4792 & 0.5208 & 0.2917 & 0.3056 & 0.5694 & 0.5486 & 0.2014 & 0.2639 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f8015b441501051e8f56d0ee593c04827efccddc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_regressor +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 7, "random_state": 0} +tau=0.054243 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_k16_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/summary_ctt.csv b/workspace/runs/summary_ctt.csv index 936e40eea6b3a9341874831df5dbd0b851318c4d..aa976776802bec4f6d23d12213b08bd1e7d586fe 100644 --- a/workspace/runs/summary_ctt.csv +++ b/workspace/runs/summary_ctt.csv @@ -5,6 +5,12 @@ CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_domina CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run +CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run +CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run +CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run +CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run +CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run +CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_context_source_envclip_k16_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_context_success_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run CTT residual base-context-observation,runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test,yes,train feature=base_context_obs,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,120,see run diff --git a/workspace/runs/summary_ctt.md b/workspace/runs/summary_ctt.md index e6585da64781de78fc93ef3b4f9d4e61eaa6ccc3..f15b7921c96c2d63ad4e5297e6f96d174077a959 100644 --- a/workspace/runs/summary_ctt.md +++ b/workspace/runs/summary_ctt.md @@ -8,6 +8,12 @@ | CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test` | | CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_train_to_test` | | CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test` | +| CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_chart_sourcecompat_obs_utility_envclip_k16_train_to_test` | +| CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_positive_envclip_k16_train_to_test` | +| CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_envclip_k16_train_to_test` | +| CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_success_weighted_reg_envclip_k16_train_to_test` | +| CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_envclip_k16_train_to_test` | +| CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_chartcompat_obs_utility_reg_envclip_k16_train_to_test` | | CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_context_source_envclip_k16_train_to_test` | | CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_context_success_train_to_test` | | CTT residual base-context-observation | train feature=base_context_obs; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test` |