diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..d0355f989479a2262503f56007328ed850a9f2f0 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test --feature-set basic --target positive_margin --k 8 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83654b67dad410a7e3a93e1d3e94ad16a30bdf28 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/config.yaml @@ -0,0 +1,13 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: basic +k: 8 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test +seed: 0 +selection_frac: 0.35 +target: positive_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e2932d47db7e93e66368f481f66e3fad3b2142c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "06375099001e740a377c402e34b77bf6974be3dc2a468974340ce41eb1b487f0", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "57df43ff4802379dfd60b5209d5a04028d721256406685decededccdbec0d49c", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..f3e7d37f9a11664a633842837a2ebb22ea4c569c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..cfbd0d95da52d2675061067b570152cacb167ef8 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics.json @@ -0,0 +1,7438 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.3541666666666667, + "fallback_rate": 0.6458333333333334, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5069444444444444, + "proposal_oracle_success": 0.4791666666666667, + "proposal_oracle_utility": 1.054027979976834, + "selected_success": 0.3055555555555556, + "selected_success_gain_over_base": 0.013888888888888888, + "selected_utility": 0.745677389996773, + "selector_regret": 0.4114182772690482, + "success_selector_gap": 0.2222222222222222, + "success_support_gap": 0.2916666666666667, + "support_gap": 0.5381004453940679 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates" + ], + "feature_set": "basic", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4007220216606498, + "fallback_rate": 0.5992779783393501, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.48375451263537905, + "proposal_oracle_success": 0.49097472924187724, + "proposal_oracle_utility": 1.0579715356210844, + "selected_success": 0.35379061371841153, + "selected_success_gain_over_base": 0.07581227436823104, + "selected_utility": 0.8292252815837908, + "selector_regret": 0.35705899309969896, + "success_selector_gap": 0.19855595667870035, + "success_support_gap": 0.20577617328519857, + "support_gap": 0.4109747204182907 + }, + "k": 8, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.36129032258064514, + "fallback_rate": 0.6387096774193548, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.33548387096774196, + "selected_success_gain_over_base": 0.05806451612903226, + "selected_utility": 0.8032603618533662, + "selector_regret": 0.3049943018208949, + "success_selector_gap": 0.16774193548387098, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "num_fit_candidates": 2216, + "num_fit_rows": 277, + "num_selection_candidates": 1240, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2781929266795631, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4702558631466304, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20163558423519135, + "selected_utility_gain_over_base": -0.3984549790620804, + "selector_regret": 0.00992196798324585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.292771638695511, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3621977651439929, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4581486946189829, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "seed": "441046872/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21190984547138214, + "selected_utility_gain_over_base": -0.4880453199148178, + "selector_regret": 0.30289267003536224, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1923524089283563, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4720823674110989, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31676346099212455, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3556138453891751, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.44029742039894637, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.00858914852142334, + "seed": "626486813/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006429394241422415, + "selected_utility_gain_over_base": 0.004499892587773502, + "selector_regret": 0.002159754280000925, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24483156204223633, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30094165208786144, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2791544623573796, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.30489884280003193, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3017327564966412, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05741874501109123, + "seed": "278008524/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.021532438695430756, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1703985370695591, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2989164896631082, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.4106108475246232, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035930830985307693, + "selected_utility_gain_over_base": 0.027663647197186947, + "selector_regret": 0.043276745826005936, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.2973160693089475, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3029974497765087, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37348877935876434, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "seed": "141984615/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2889911485479255, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.48545416581659245, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01579594612121582, + "selected_utility_gain_over_base": -0.06733909994363785, + "selector_regret": 0.07246536016464233, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.3085909839571368, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27576249231752253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27574476828623157, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.4450432151768497, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.019754549488425255, + "selected_utility_gain_over_base": -0.07324294187128544, + "selector_regret": 0.02163984440267086, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3930751315324732, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.3303509116866848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3018717520234185, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.37672502201482544, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.015449045225977898, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3460545899296898, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24804890411536307, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5804248011896158, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.42792806960140634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "seed": "1324642411/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08104583621025085, + "selected_utility_gain_over_base": -0.025424689054489136, + "selector_regret": 0.09088736772537231, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.48888892485386093, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "seed": "1790188787/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06870591640472412, + "selected_utility_gain_over_base": -0.056061550974845886, + "selector_regret": 0.029213324189186096, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.49362379866268385, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06716175377368927, + "selected_utility_gain_over_base": -1.9328382462263107, + "selector_regret": 1.9328382462263107, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.44036449124726545, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.6689300537109375e-06, + "selected_utility_gain_over_base": 1.4454126358032227e-06, + "selector_regret": 1.9999983310699463, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.39508824247975904, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2680360950974473, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2996766962199408, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2748700135153204, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30129083453678585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29567751322199776, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2838401268071768, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8090915828943253, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32375809904022324, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30291602696952474, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2996766962199408, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28607089947933784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2864812386218567, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3755843205364714, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6900332570075989, + "selected_utility_gain_over_base": -0.03357207775115967, + "selector_regret": 1.3099667429924011, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3755843205364714, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21116642653942108, + "selected_utility_gain_over_base": -0.38892413675785065, + "selector_regret": 0.3764408975839615, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36117233126140574, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "seed": "1073161778/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3622665827290821, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3622665827290821, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.26480661258416843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4339821423007203, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28276498582888665, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12861795723438263, + "seed": "767747424/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8713820427656174, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.506786610273882, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.016585012897849083, + "selected_utility_gain_over_base": -0.07731805182993412, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.4730304978528036, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00289076566696167, + "selected_utility_gain_over_base": 0.000961264013312757, + "selector_regret": 0.0070051951333880424, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29654516341737563, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13936175405979156, + "seed": "2054899126/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8606382459402084, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3059562851528022, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.4367530798262121, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13429443538188934, + "selected_utility_gain_over_base": 0.05730593949556351, + "selector_regret": 0.07473599910736084, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.45846266415307685, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04821258783340454, + "selected_utility_gain_over_base": 0.012326281517744064, + "selector_regret": 0.027136601507663727, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.35433654964210193, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.580314676746412, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05121459439396858, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.020476004108786583, + "selected_utility_gain_over_base": 0.012208820320665836, + "selector_regret": 0.030738590285182, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026529457420110703, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.30883462632151115, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28994114758485556, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.40137543246827484, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.037011127918958664, + "selected_utility_gain_over_base": -0.06582311913371086, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3556038068424024, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.43111137474421807, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017789265140891075, + "selected_utility_gain_over_base": -0.0653457809239626, + "selector_regret": 0.04962191917002201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.4754238921971224, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.010783910751342773, + "selected_utility_gain_over_base": -0.04333903267979622, + "selector_regret": 0.03199801966547966, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30671589390658466, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "seed": "1375492786/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2963268387066977, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.3184181476758302, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018512330949306488, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4099616036201487, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.365539554390633, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3622665827290821, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.3606900164943357, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2999793737204269, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30973485490721986, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5938315939300237, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02717691659927368, + "selected_utility_gain_over_base": -0.008975863456726074, + "selector_regret": 1.9728230834007263, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3921674767850758, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "seed": "1324642411/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10393516719341278, + "selected_utility_gain_over_base": -0.0025353580713272095, + "selector_regret": 0.0797557383775711, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3755843205364714, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "seed": "1790188787/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11081808805465698, + "selected_utility_gain_over_base": -0.013949379324913025, + "selector_regret": 0.019496619701385498, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5844950408029461, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4239855826799761, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "seed": "509430392/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.564621925354004e-06, + "selected_utility_gain_over_base": 1.341104507446289e-06, + "selector_regret": 6.008148193359375e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5930611397299111, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29770620738185305, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32254979421285396, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "seed": "655884605/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0021464377641677856, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34929520973604355, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3882270730285144, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18939486145973206, + "selected_utility_gain_over_base": -0.6347416341304779, + "selector_regret": 1.810605138540268, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32003263953794914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36516876704401185, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34860626835972375, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "seed": "1151410346/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5878906548023224, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3015169653629154, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3954257212909892, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3570017249937056, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3060895394094739, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34895677424461957, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32005010012998786, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "seed": "307068879/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3457649211345198, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2897073440884107, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "seed": "608016290/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3550679344666073, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "seed": "441046872/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013073742389678955, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "task_id": "LiftPegUpright-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.25057469871698324, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.47673474284532225, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29798110129084976, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18884079158306122, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.027154862880706787, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8111592084169388, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.3809741420013, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "seed": "1461069713/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06527054309844971, + "selected_utility_gain_over_base": -0.028632521629333496, + "selector_regret": 0.034432850778102875, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.39813777059860467, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.026186764240264893, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02248450182378292, + "selected_utility_gain_over_base": 0.020555000170134008, + "selector_regret": 0.0037022624164819717, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22723394632339478, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2636770909966693, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09105253219604492, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.908947467803955, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3435013909594266, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.32925314383151205, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3460742067973036, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3440215189574023, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.35381528262382067, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.3587698011321242, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2844961787136048, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3033646357953552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "seed": "141984615/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29389385208700486, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.42062384206560416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0756516233086586, + "selected_utility_gain_over_base": -0.007483422756195068, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.42450207308549714, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04602654650807381, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035472434014081955, + "selected_utility_gain_over_base": -0.018650509417057037, + "selector_regret": 0.010554112493991852, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15108097717165947, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2818032504385155, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2728649751609838, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.3816360107614827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04953896999359131, + "seed": "750448296/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03674823045730591, + "selected_utility_gain_over_base": -0.056249260902404785, + "selector_regret": 0.0127907395362854, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17445051670074463, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.48517873944507806, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.39450829110578517, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06004788354039192, + "selected_utility_gain_over_base": -1.939952116459608, + "selector_regret": 0.06505399569869041, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3955843378144643, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.45827775488717093, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "seed": "1549324805/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03407943248748779, + "selected_utility_gain_over_base": -0.07043540477752686, + "selector_regret": 0.056848861277103424, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35054441428919914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24680178437921813, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5137779251382697, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31840010979467775, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3785435132544391, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09652663767337799, + "selected_utility_gain_over_base": -0.028240829706192017, + "selector_regret": 1.903473362326622, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5690133983051641, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08501242101192474, + "selected_utility_gain_over_base": -1.9149875789880753, + "selector_regret": 1.9149875789880753, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.329217542757152, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 5.200505256652832e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.41143741228233793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0876526832580566e-05, + "selected_utility_gain_over_base": -0.08701319992542267, + "selector_regret": 1.9999791234731674, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27387772560678275, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31908267001192103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28994114758485556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4763906292645175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1460946798324585, + "selected_utility_gain_over_base": -0.6780418157577515, + "selector_regret": 1.8539053201675415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2903038021197549, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "seed": "1509980664/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.344497387758629, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "seed": "211245842/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02432851493358612, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3657069596994634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "seed": "1151410346/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2829121416380268, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5158707077594902, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37134486499196434, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27674986209929975, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3741559563273496, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "max_depth": 3, + "min_samples_leaf": 8, + "n_estimators": 128, + "n_jobs": 1, + "random_state": 0 + }, + "selected_model_type": "rf_regressor", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.36129032258064514, + "fallback_rate": 0.6387096774193548, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.33548387096774196, + "selected_success_gain_over_base": 0.05806451612903226, + "selected_utility": 0.8032603618533662, + "selector_regret": 0.3049943018208949, + "success_selector_gap": 0.16774193548387098, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4395833333333333, + "low": 0.2777777777777778, + "mean": 0.35347222222222224, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.35347222222222224, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.0, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6109090909090908, + "low": 0.09624242424242424, + "mean": 0.3273535353535354, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3273535353535354, + "per_task": { + "LiftPegUpright-v1": 0.26666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4395833333333333, + "low": 0.2777777777777778, + "mean": 0.35347222222222224, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.35347222222222224, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.0, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6109090909090908, + "low": 0.09624242424242424, + "mean": 0.3273535353535354, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3273535353535354, + "per_task": { + "LiftPegUpright-v1": 0.26666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.8888888888888888, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.4305555555555556, + "low": 0.2777777777777778, + "mean": 0.3541666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7222222222222222, + "low": 0.5604166666666667, + "mean": 0.6465277777777778, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6465277777777778, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 1.0, + "767747424/train1": 0.5, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9037575757575758, + "low": 0.38909090909090904, + "mean": 0.6726464646464647, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6726464646464647, + "per_task": { + "LiftPegUpright-v1": 0.7333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.64, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.8787878787878788 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.7222222222222222, + "low": 0.5694444444444444, + "mean": 0.6458333333333334, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5770833333333333, + "low": 0.40277777777777773, + "mean": 0.48819444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.48819444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6698181818181819, + "low": 0.20315151515151517, + "mean": 0.452040404040404, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.452040404040404, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.44, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.5757575757575758 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.5902777777777778, + "low": 0.4305555555555556, + "mean": 0.5069444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3765648581500766, + "low": 0.34899999570769163, + "mean": 0.36230770804478146, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.36230770804478146, + "per_seed": { + "1073161778/train0": 0.292771638695511, + "1073161778/train1": 0.36117233126140574, + "1073161778/train2": 0.3457649211345198, + "1113371261/train0": 0.2748700135153204, + "1113371261/train1": 0.34929520973604355, + "1113371261/train2": 0.28994114758485556, + "1151410346/train0": 0.32375809904022324, + "1151410346/train1": 0.34860626835972375, + "1151410346/train2": 0.3657069596994634, + "1261930456/train0": 0.49362379866268385, + "1261930456/train1": 0.5844950408029461, + "1261930456/train2": 0.5690133983051641, + "1324133122/train0": 0.30291602696952474, + "1324133122/train1": 0.3015169653629154, + "1324133122/train2": 0.2829121416380268, + "1324642411/train0": 0.42792806960140634, + "1324642411/train1": 0.3921674767850758, + "1324642411/train2": 0.31840010979467775, + "1375492786/train0": 0.27576249231752253, + "1375492786/train1": 0.30671589390658466, + "1375492786/train2": 0.2818032504385155, + "141984615/train0": 0.37348877935876434, + "141984615/train1": 0.40137543246827484, + "141984615/train2": 0.3033646357953552, + "1461069713/train0": 0.3556138453891751, + "1461069713/train1": 0.506786610273882, + "1461069713/train2": 0.3809741420013, + "1480856810/train0": 0.39508824247975904, + "1480856810/train1": 0.5930611397299111, + "1480856810/train2": 0.41143741228233793, + "1509980664/train0": 0.29567751322199776, + "1509980664/train1": 0.32003263953794914, + "1509980664/train2": 0.2903038021197549, + "1522272686/train0": 0.1923524089283563, + "1522272686/train1": 0.26480661258416843, + "1522272686/train2": 0.25057469871698324, + "1549324805/train0": 0.37672502201482544, + "1549324805/train1": 0.3606900164943357, + "1549324805/train2": 0.45827775488717093, + "1578990937/train0": 0.30129083453678585, + "1578990937/train1": 0.3882270730285144, + "1578990937/train2": 0.4763906292645175, + "1747178030/train0": 0.4720823674110989, + "1747178030/train1": 0.4339821423007203, + "1747178030/train2": 0.47673474284532225, + "1790188787/train0": 0.48888892485386093, + "1790188787/train1": 0.3755843205364714, + "1790188787/train2": 0.3785435132544391, + "1822210471/train0": 0.3460545899296898, + "1822210471/train1": 0.2999793737204269, + "1822210471/train2": 0.35054441428919914, + "1845489013/train0": 0.2989164896631082, + "1845489013/train1": 0.35433654964210193, + "1845489013/train2": 0.3440215189574023, + "1999022574/train0": 0.2864812386218567, + "1999022574/train1": 0.3060895394094739, + "1999022574/train2": 0.27674986209929975, + "2012211287/train0": 0.2781929266795631, + "2012211287/train1": 0.3755843205364714, + "2012211287/train2": 0.34895677424461957, + "2038594191/train0": 0.27574476828623157, + "2038594191/train1": 0.2963268387066977, + "2038594191/train2": 0.2728649751609838, + "2054899126/train0": 0.3470083918101673, + "2054899126/train1": 0.3532533835187622, + "2054899126/train2": 0.3744279152208737, + "2061635888/train0": 0.2680360950974473, + "2061635888/train1": 0.29770620738185305, + "2061635888/train2": 0.27387772560678275, + "211245842/train0": 0.2838401268071768, + "211245842/train1": 0.36516876704401185, + "211245842/train2": 0.344497387758629, + "278008524/train0": 0.3018022542600298, + "278008524/train1": 0.4103646234410795, + "278008524/train2": 0.3708292723058839, + "307068879/train0": 0.4702558631466304, + "307068879/train1": 0.3755843205364714, + "307068879/train2": 0.32005010012998786, + "412102563/train0": 0.28607089947933784, + "412102563/train1": 0.3570017249937056, + "412102563/train2": 0.37134486499196434, + "440577711/train0": 0.5804248011896158, + "440577711/train1": 0.5938315939300237, + "440577711/train2": 0.5137779251382697, + "441046872/train0": 0.4581486946189829, + "441046872/train1": 0.3622665827290821, + "441046872/train2": 0.3550679344666073, + "509430392/train0": 0.44036449124726545, + "509430392/train1": 0.4239855826799761, + "509430392/train2": 0.329217542757152, + "585904130/train0": 0.3029974497765087, + "585904130/train1": 0.28994114758485556, + "585904130/train2": 0.2844961787136048, + "608016290/train0": 0.3621977651439929, + "608016290/train1": 0.3622665827290821, + "608016290/train2": 0.2897073440884107, + "626486813/train0": 0.4254541339617848, + "626486813/train1": 0.5266725872996079, + "626486813/train2": 0.37597652661121267, + "636680137/train0": 0.48545416581659245, + "636680137/train1": 0.43111137474421807, + "636680137/train2": 0.42062384206560416, + "655884605/train0": 0.2996766962199408, + "655884605/train1": 0.32254979421285396, + "655884605/train2": 0.31908267001192103, + "672328262/train0": 0.2791544623573796, + "672328262/train1": 0.3059562851528022, + "672328262/train2": 0.3435013909594266, + "701667486/train0": 0.2996766962199408, + "701667486/train1": 0.3954257212909892, + "701667486/train2": 0.5158707077594902, + "747260437/train0": 0.3303509116866848, + "747260437/train1": 0.365539554390633, + "747260437/train2": 0.39450829110578517, + "750448296/train0": 0.3273610892800461, + "750448296/train1": 0.32791893647515086, + "750448296/train2": 0.3074438824092352, + "767747424/train0": 0.30689233926456017, + "767747424/train1": 0.37594414604343307, + "767747424/train2": 0.35262652983499576 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.41377120897638753, + "low": 0.27845274198613496, + "mean": 0.347859569335541, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.347859569335541, + "per_task": { + "LiftPegUpright-v1": 0.35719920667608923, + "PickCube-v1": 0.23591124007650266, + "PullCube-v1": 0.357845382167158, + "PushCube-v1": 0.4616574102239464, + "StackCube-v1": 0.32668460753400885 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.3740473511657917, + "low": 0.34824139741794924, + "mean": 0.3610732702469515, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5798611111111112, + "low": 0.4131944444444445, + "mean": 0.4972222222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4972222222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5842424242424242, + "low": 0.18424242424242424, + "mean": 0.41090909090909095, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.41090909090909095, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.45454545454545453 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.5555555555555556, + "low": 0.3958333333333333, + "mean": 0.4791666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2489316311703687, + "low": 0.9516223322328491, + "mean": 1.0964438248309307, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.0964438248309307, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 0.7071996331214905, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 0.8138919472694397, + "1151410346/train2": 0.21912741661071777, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.17193320393562317, + "1324642411/train1": 0.1836909055709839, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.08251869678497314, + "1375492786/train2": 2.0, + "141984615/train0": 0.007291515823453665, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.030228236690163612, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09970339387655258, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.8162217140197754, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.0415392704308033, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.09092829376459122, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.09791924059391022, + "1790188787/train1": 0.13031470775604248, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 1.0696808770298958, + "2054899126/train2": 1.0455262660980225, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.21523693203926086, + "278008524/train0": 1.0287093725055456, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.22727864980697632, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.5148025155067444, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7130289077758789, + "509430392/train0": 2.0, + "509430392/train1": 6.164610385894775e-05, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.6825780272483826, + "626486813/train0": 0.043898362666368484, + "626486813/train1": 0.030555277597159147, + "626486813/train2": 0.03856666944921017, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.1651531159877777, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.05267423391342163, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.349846323331197, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 0.5951075805351138, + "767747424/train2": 0.6134321661666036 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2413418491681416, + "low": 0.5217755792898437, + "mean": 0.9521664379400512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9521664379400512, + "per_task": { + "LiftPegUpright-v1": 1.2244940986235937, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 0.9816135169876119, + "PushCube-v1": 1.365773618221283, + "StackCube-v1": 1.0546941955884297 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.196670868741219, + "low": 0.9012325454645583, + "mean": 1.054027979976834, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 3.5680555555555555, + "low": 2.69375, + "mean": 3.127777777777778, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.127777777777778, + "per_seed": { + "1073161778/train0": 4.0, + "1073161778/train1": 3.0, + "1073161778/train2": 2.0, + "1113371261/train0": 1.0, + "1113371261/train1": 2.0, + "1113371261/train2": 4.0, + "1151410346/train0": 6.0, + "1151410346/train1": 7.0, + "1151410346/train2": 6.0, + "1261930456/train0": 3.0, + "1261930456/train1": 2.0, + "1261930456/train2": 4.0, + "1324133122/train0": 2.0, + "1324133122/train1": 6.0, + "1324133122/train2": 2.0, + "1324642411/train0": 3.0, + "1324642411/train1": 1.0, + "1324642411/train2": 7.0, + "1375492786/train0": 7.0, + "1375492786/train1": 2.0, + "1375492786/train2": 7.0, + "141984615/train0": 2.0, + "141984615/train1": 0.0, + "141984615/train2": 4.0, + "1461069713/train0": 4.0, + "1461069713/train1": 4.0, + "1461069713/train2": 3.0, + "1480856810/train0": 2.0, + "1480856810/train1": 4.0, + "1480856810/train2": 1.0, + "1509980664/train0": 6.0, + "1509980664/train1": 7.0, + "1509980664/train2": 5.0, + "1522272686/train0": 3.0, + "1522272686/train1": 4.0, + "1522272686/train2": 3.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 7.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 6.0, + "1747178030/train2": 4.0, + "1790188787/train0": 0.0, + "1790188787/train1": 5.0, + "1790188787/train2": 7.0, + "1822210471/train0": 3.0, + "1822210471/train1": 3.0, + "1822210471/train2": 7.0, + "1845489013/train0": 7.0, + "1845489013/train1": 4.0, + "1845489013/train2": 7.0, + "1999022574/train0": 2.0, + "1999022574/train1": 4.0, + "1999022574/train2": 4.0, + "2012211287/train0": 5.0, + "2012211287/train1": 7.0, + "2012211287/train2": 6.0, + "2038594191/train0": 5.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 3.0, + "2054899126/train2": 0.0, + "2061635888/train0": 2.0, + "2061635888/train1": 7.0, + "2061635888/train2": 6.0, + "211245842/train0": 3.0, + "211245842/train1": 6.0, + "211245842/train2": 0.0, + "278008524/train0": 5.0, + "278008524/train1": 0.5, + "278008524/train2": 3.5, + "307068879/train0": 3.0, + "307068879/train1": 5.0, + "307068879/train2": 5.0, + "412102563/train0": 3.0, + "412102563/train1": 7.0, + "412102563/train2": 7.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 2.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 5.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 2.0, + "608016290/train0": 5.0, + "608016290/train1": 1.0, + "608016290/train2": 4.0, + "626486813/train0": 2.0, + "626486813/train1": 1.5, + "626486813/train2": 0.0, + "636680137/train0": 2.0, + "636680137/train1": 1.0, + "636680137/train2": 2.0, + "655884605/train0": 6.0, + "655884605/train1": 5.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 7.0, + "701667486/train1": 7.0, + "701667486/train2": 0.0, + "747260437/train0": 5.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 3.6666666666666665, + "750448296/train1": 2.6666666666666665, + "750448296/train2": 4.0, + "767747424/train0": 3.25, + "767747424/train1": 0.75, + "767747424/train2": 2.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 3.7769696969696964, + "low": 2.6595555555555555, + "mean": 3.218262626262626, + "n": 5, + "num_samples": 1000 + }, + "mean": 3.218262626262626, + "per_task": { + "LiftPegUpright-v1": 3.533333333333333, + "PickCube-v1": 3.3333333333333335, + "PullCube-v1": 2.4266666666666667, + "PushCube-v1": 2.5555555555555554, + "StackCube-v1": 4.242424242424242 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.4166666666666665, + "low": 2.5902777777777777, + "mean": 2.9930555555555554, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.40208333333333335, + "low": 0.23958333333333334, + "mean": 0.31875, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.31875, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.31066666666666665, + "low": 0.09555555555555556, + "mean": 0.22210101010101013, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.22210101010101013, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3819444444444444, + "low": 0.2361111111111111, + "mean": 0.3055555555555556, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.058333333333333334, + "low": -0.03333333333333333, + "mean": 0.0125, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.0125, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": -1.0, + "1261930456/train1": 0.0, + "1261930456/train2": -1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.06666666666666667, + "low": 0.0, + "mean": 0.02222222222222222, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.02222222222222222, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": 0.0 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.05555555555555555, + "low": -0.020833333333333332, + "mean": 0.013888888888888888, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9512949052480206, + "low": 0.639972912685092, + "mean": 0.787043470071189, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.787043470071189, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 0.06716175377368927, + "1261930456/train1": 2.0, + "1261930456/train2": 0.08501242101192474, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.08104583621025085, + "1324642411/train1": 0.10393516719341278, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.06527054309844971, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0876526832580566e-05, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.0260902252048254, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.03407943248748779, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.18939486145973206, + "1578990937/train2": 0.1460946798324585, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.06870591640472412, + "1790188787/train1": 0.11081808805465698, + "1790188787/train2": 0.09652663767337799, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.6900332570075989, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.048898130655288696, + "278008524/train2": 1.0179431531578302, + "307068879/train0": 0.20163558423519135, + "307068879/train1": 0.21116642653942108, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 0.02717691659927368, + "440577711/train2": 2.0, + "441046872/train0": 0.21190984547138214, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 1.6689300537109375e-06, + "509430392/train1": 1.564621925354004e-06, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.021180112613365054, + "626486813/train1": 0.011683384887874126, + "626486813/train2": 0.015375842805951834, + "636680137/train0": 0.01579594612121582, + "636680137/train1": 0.017789265140891075, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 0.06004788354039192, + "750448296/train0": 0.09317883538703124, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.09884339570999146, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5766910687088966, + "767747424/train2": 0.5685367146506906 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8060935289052626, + "low": 0.6197258207533094, + "mean": 0.70972903242997, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.70972903242997, + "per_task": { + "LiftPegUpright-v1": 0.8626041044791539, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7684198118560016, + "PushCube-v1": 0.5970487553212378, + "StackCube-v1": 0.7366133878628413 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8881475064618927, + "low": 0.608154530814823, + "mean": 0.745677389996773, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.0899972230681063, + "low": -0.0927519937556806, + "mean": -0.0030745922266052932, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.0030745922266052932, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": -1.9328382462263107, + "1261930456/train1": 0.0, + "1261930456/train2": -1.9149875789880753, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": -0.025424689054489136, + "1324642411/train1": -0.0025353580713272095, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": -0.06582311913371086, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": -0.07731805182993412, + "1461069713/train2": -0.028632521629333496, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.9129659235477448, + "1480856810/train2": -0.08701319992542267, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": -0.07842461206018925, + "1549324805/train1": 0.0, + "1549324805/train2": -0.07043540477752686, + "1578990937/train0": 0.0, + "1578990937/train1": -0.6347416341304779, + "1578990937/train2": -0.6780418157577515, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": -0.056061550974845886, + "1790188787/train1": -0.013949379324913025, + "1790188787/train2": -0.028240829706192017, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": -0.03357207775115967, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.006163140758872032, + "278008524/train2": 0.9752081632614136, + "307068879/train0": -0.3984549790620804, + "307068879/train1": -0.38892413675785065, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": -0.008975863456726074, + "440577711/train2": 1.9638472199440002, + "441046872/train0": -0.4880453199148178, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.4454126358032227e-06, + "509430392/train1": 1.341104507446289e-06, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.016081769892480224, + "626486813/train1": 0.006585042166989297, + "626486813/train2": 0.010277500085067004, + "636680137/train0": -0.06733909994363785, + "636680137/train1": -0.0653457809239626, + "636680137/train2": -0.007483422756195068, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": -1.939952116459608, + "750448296/train0": -0.024414313957095146, + "750448296/train1": 0.0, + "750448296/train2": -0.01874975363413493, + "767747424/train0": 0.0, + "767747424/train1": 0.003491726703941822, + "767747424/train2": -0.004662627354264259 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.11348035271959662, + "low": -0.05506012028641999, + "mean": 0.014169235255954921, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.014169235255954921, + "per_task": { + "LiftPegUpright-v1": -0.0872664342323939, + "PickCube-v1": 0.0, + "PullCube-v1": -0.006750649367459119, + "PushCube-v1": 0.2046445765429073, + "StackCube-v1": -0.03978131666327968 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.08544879256482495, + "low": -0.07008276534098615, + "mean": 0.0038578035544358296, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5373593460238124, + "low": 0.2975059330023618, + "mean": 0.40926073354914677, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.40926073354914677, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 0.5878906548023224, + "1151410346/train2": 0.0, + "1261930456/train0": 1.9328382462263107, + "1261930456/train1": 0.0, + "1261930456/train2": 1.9149875789880753, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.09088736772537231, + "1324642411/train1": 0.0797557383775711, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.034432850778102875, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.9999791234731674, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.015449045225977898, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.056848861277103424, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.810605138540268, + "1578990937/train2": 1.8539053201675415, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.029213324189186096, + "1790188787/train1": 0.019496619701385498, + "1790188787/train2": 1.903473362326622, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.3099667429924011, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8090915828943253, + "211245842/train1": 0.02204875648021698, + "211245842/train2": 0.02432851493358612, + "278008524/train0": 0.985974382609129, + "278008524/train1": 0.9887764640152454, + "278008524/train2": 0.11800575815141201, + "307068879/train0": 0.00992196798324585, + "307068879/train1": 0.3764408975839615, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.9728230834007263, + "440577711/train2": 0.0, + "441046872/train0": 0.30289267003536224, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.013073742389678955, + "509430392/train0": 1.9999983310699463, + "509430392/train1": 6.008148193359375e-05, + "509430392/train2": 5.200505256652832e-06, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.02271825005300343, + "626486813/train1": 0.01887189270928502, + "626486813/train2": 0.023190826643258333, + "636680137/train0": 0.07246536016464233, + "636680137/train1": 0.04962191917002201, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0021464377641677856, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.06505399569869041, + "750448296/train0": 0.6594814925144116, + "750448296/train1": 1.2529101247588794, + "750448296/train2": 1.2510029276212056, + "767747424/train0": 0.4779601227492094, + "767747424/train1": 0.026683504693210125, + "767747424/train2": 0.04489545151591301 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5993976444086118, + "low": 0.15964173394622225, + "mean": 0.38706884318729384, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.38706884318729384, + "per_task": { + "LiftPegUpright-v1": 0.47580571969350177, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3684106833053132, + "PushCube-v1": 0.7687248629000452, + "StackCube-v1": 0.32240295003760944 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.5321950333762087, + "low": 0.2979386186699331, + "mean": 0.4114182772690482, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3027777777777778, + "low": 0.1590277777777778, + "mean": 0.22430555555555556, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.22430555555555556, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3262222222222222, + "low": 0.10303030303030303, + "mean": 0.21814141414141414, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.21814141414141414, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.18666666666666668, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.18181818181818182 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.2916666666666667, + "low": 0.1597222222222222, + "mean": 0.2222222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39861111111111114, + "low": 0.2375, + "mean": 0.3194444444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3194444444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7757575757575758, + "low": 0.23733333333333334, + "mean": 0.501090909090909, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.501090909090909, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.09333333333333334, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.5454545454545454 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7096626987680793, + "low": 0.43689682670713714, + "mean": 0.5784295155162302, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5784295155162302, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.2928003668785095, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 1.1861080527305603, + "1151410346/train2": 1.7808725833892822, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8280667960643768, + "1324642411/train1": 1.8163090944290161, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.9174813032150269, + "1375492786/train2": 0.0, + "141984615/train0": 0.27848575497046113, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2555490341037512, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.059639059007167816, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.1837782859802246, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.255916815251112, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.20652779191732407, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.9020807594060898, + "1790188787/train1": 1.8696852922439575, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.9303191229701042, + "2054899126/train2": 0.9544737339019775, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7847630679607391, + "278008524/train0": 0.08519926853477955, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7727213501930237, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.4851974844932556, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.286971092224121, + "509430392/train0": 0.0, + "509430392/train1": 1.999938353896141, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.3174219727516174, + "626486813/train0": 0.12241578102111816, + "626486813/train1": 0.13502710359171033, + "626486813/train2": 0.1270157117396593, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.8348468840122223, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.22598901391029358, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05815017223358154, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.5129444571211934, + "767747424/train2": 0.49461987148970366 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.4353523810704547, + "low": 0.47464505295703807, + "mean": 0.8912108774023751, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8912108774023751, + "per_task": { + "LiftPegUpright-v1": 0.7755059013764064, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.23527305972451965, + "PushCube-v1": 0.634226381778717, + "StackCube-v1": 0.9453058044115702 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.6574370106811531, + "low": 0.4129596278103741, + "mean": 0.5381004453940679, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3741559563273497, + "low": 0.3741559563273497, + "mean": 0.3741559563273497, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3741559563273497, + "per_seed": { + "1073161778/train0": 0.3741559563273496, + "1073161778/train1": 0.3741559563273496, + "1073161778/train2": 0.3741559563273496, + "1113371261/train0": 0.3741559563273496, + "1113371261/train1": 0.3741559563273496, + "1113371261/train2": 0.3741559563273496, + "1151410346/train0": 0.3741559563273496, + "1151410346/train1": 0.3741559563273496, + "1151410346/train2": 0.3741559563273496, + "1261930456/train0": 0.3741559563273496, + "1261930456/train1": 0.3741559563273496, + "1261930456/train2": 0.3741559563273496, + "1324133122/train0": 0.3741559563273496, + "1324133122/train1": 0.3741559563273496, + "1324133122/train2": 0.3741559563273496, + "1324642411/train0": 0.3741559563273496, + "1324642411/train1": 0.3741559563273496, + "1324642411/train2": 0.3741559563273496, + "1375492786/train0": 0.3741559563273496, + "1375492786/train1": 0.3741559563273496, + "1375492786/train2": 0.3741559563273496, + "141984615/train0": 0.3741559563273496, + "141984615/train1": 0.3741559563273496, + "141984615/train2": 0.3741559563273496, + "1461069713/train0": 0.3741559563273496, + "1461069713/train1": 0.3741559563273496, + "1461069713/train2": 0.3741559563273496, + "1480856810/train0": 0.3741559563273496, + "1480856810/train1": 0.3741559563273496, + "1480856810/train2": 0.3741559563273496, + "1509980664/train0": 0.3741559563273496, + "1509980664/train1": 0.3741559563273496, + "1509980664/train2": 0.3741559563273496, + "1522272686/train0": 0.3741559563273496, + "1522272686/train1": 0.3741559563273496, + "1522272686/train2": 0.3741559563273496, + "1549324805/train0": 0.3741559563273496, + "1549324805/train1": 0.3741559563273496, + "1549324805/train2": 0.3741559563273496, + "1578990937/train0": 0.3741559563273496, + "1578990937/train1": 0.3741559563273496, + "1578990937/train2": 0.3741559563273496, + "1747178030/train0": 0.3741559563273496, + "1747178030/train1": 0.3741559563273496, + "1747178030/train2": 0.3741559563273496, + "1790188787/train0": 0.3741559563273496, + "1790188787/train1": 0.3741559563273496, + "1790188787/train2": 0.3741559563273496, + "1822210471/train0": 0.3741559563273496, + "1822210471/train1": 0.3741559563273496, + "1822210471/train2": 0.3741559563273496, + "1845489013/train0": 0.3741559563273496, + "1845489013/train1": 0.3741559563273496, + "1845489013/train2": 0.3741559563273496, + "1999022574/train0": 0.3741559563273496, + "1999022574/train1": 0.3741559563273496, + "1999022574/train2": 0.3741559563273496, + "2012211287/train0": 0.3741559563273496, + "2012211287/train1": 0.3741559563273496, + "2012211287/train2": 0.3741559563273496, + "2038594191/train0": 0.3741559563273496, + "2038594191/train1": 0.3741559563273496, + "2038594191/train2": 0.3741559563273496, + "2054899126/train0": 0.3741559563273496, + "2054899126/train1": 0.3741559563273496, + "2054899126/train2": 0.3741559563273496, + "2061635888/train0": 0.3741559563273496, + "2061635888/train1": 0.3741559563273496, + "2061635888/train2": 0.3741559563273496, + "211245842/train0": 0.3741559563273496, + "211245842/train1": 0.3741559563273496, + "211245842/train2": 0.3741559563273496, + "278008524/train0": 0.3741559563273496, + "278008524/train1": 0.3741559563273496, + "278008524/train2": 0.3741559563273496, + "307068879/train0": 0.3741559563273496, + "307068879/train1": 0.3741559563273496, + "307068879/train2": 0.3741559563273496, + "412102563/train0": 0.3741559563273496, + "412102563/train1": 0.3741559563273496, + "412102563/train2": 0.3741559563273496, + "440577711/train0": 0.3741559563273496, + "440577711/train1": 0.3741559563273496, + "440577711/train2": 0.3741559563273496, + "441046872/train0": 0.3741559563273496, + "441046872/train1": 0.3741559563273496, + "441046872/train2": 0.3741559563273496, + "509430392/train0": 0.3741559563273496, + "509430392/train1": 0.3741559563273496, + "509430392/train2": 0.3741559563273496, + "585904130/train0": 0.3741559563273496, + "585904130/train1": 0.3741559563273496, + "585904130/train2": 0.3741559563273496, + "608016290/train0": 0.3741559563273496, + "608016290/train1": 0.3741559563273496, + "608016290/train2": 0.3741559563273496, + "626486813/train0": 0.3741559563273496, + "626486813/train1": 0.3741559563273496, + "626486813/train2": 0.3741559563273496, + "636680137/train0": 0.3741559563273496, + "636680137/train1": 0.3741559563273496, + "636680137/train2": 0.3741559563273496, + "655884605/train0": 0.3741559563273496, + "655884605/train1": 0.3741559563273496, + "655884605/train2": 0.3741559563273496, + "672328262/train0": 0.3741559563273496, + "672328262/train1": 0.3741559563273496, + "672328262/train2": 0.3741559563273496, + "701667486/train0": 0.3741559563273496, + "701667486/train1": 0.3741559563273496, + "701667486/train2": 0.3741559563273496, + "747260437/train0": 0.3741559563273496, + "747260437/train1": 0.3741559563273496, + "747260437/train2": 0.3741559563273496, + "750448296/train0": 0.3741559563273496, + "750448296/train1": 0.3741559563273496, + "750448296/train2": 0.3741559563273496, + "767747424/train0": 0.3741559563273496, + "767747424/train1": 0.3741559563273496, + "767747424/train2": 0.3741559563273496 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3741559563273497, + "low": 0.37415595632734966, + "mean": 0.3741559563273497, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3741559563273497, + "per_task": { + "LiftPegUpright-v1": 0.3741559563273497, + "PickCube-v1": 0.3741559563273496, + "PullCube-v1": 0.3741559563273497, + "PushCube-v1": 0.3741559563273497, + "StackCube-v1": 0.3741559563273497 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.3741559563273497, + "low": 0.3741559563273497, + "mean": 0.3741559563273497, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "positive_margin", + "tau": 0.3741559563273496 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..84665a6a4386455799052052a7e974cc4eded3f8 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.292771638695511, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36117233126140574, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "tau": 0.3741559563273496 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3457649211345198, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2748700135153204, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.3741559563273496 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34929520973604355, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.3741559563273496 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28994114758485556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.3741559563273496 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32375809904022324, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.3741559563273496 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34860626835972375, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5878906548023224, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "tau": 0.3741559563273496 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3657069596994634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "tau": 0.3741559563273496 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.49362379866268385, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06716175377368927, + "selected_utility_gain_over_base": -1.9328382462263107, + "selector_regret": 1.9328382462263107, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5844950408029461, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5690133983051641, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.08501242101192474, + "selected_utility_gain_over_base": -1.9149875789880753, + "selector_regret": 1.9149875789880753, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30291602696952474, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.3741559563273496 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3015169653629154, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.3741559563273496 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2829121416380268, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.3741559563273496 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.42792806960140634, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08104583621025085, + "selected_utility_gain_over_base": -0.025424689054489136, + "selector_regret": 0.09088736772537231, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "tau": 0.3741559563273496 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3921674767850758, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10393516719341278, + "selected_utility_gain_over_base": -0.0025353580713272095, + "selector_regret": 0.0797557383775711, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "tau": 0.3741559563273496 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31840010979467775, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27576249231752253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30671589390658466, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "tau": 0.3741559563273496 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2818032504385155, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.37348877935876434, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "tau": 0.3741559563273496 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.40137543246827484, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.037011127918958664, + "selected_utility_gain_over_base": -0.06582311913371086, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.3741559563273496 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.3033646357953552, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "tau": 0.3741559563273496 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3556138453891751, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.3741559563273496 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.506786610273882, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.016585012897849083, + "selected_utility_gain_over_base": -0.07731805182993412, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.3741559563273496 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.3809741420013, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06527054309844971, + "selected_utility_gain_over_base": -0.028632521629333496, + "selector_regret": 0.034432850778102875, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "tau": 0.3741559563273496 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.39508824247975904, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5930611397299111, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.41143741228233793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0876526832580566e-05, + "selected_utility_gain_over_base": -0.08701319992542267, + "selector_regret": 1.9999791234731674, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29567751322199776, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32003263953794914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2903038021197549, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "tau": 0.3741559563273496 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1923524089283563, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.3741559563273496 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.26480661258416843, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.3741559563273496 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.25057469871698324, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.3741559563273496 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.37672502201482544, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0260902252048254, + "selected_utility_gain_over_base": -0.07842461206018925, + "selector_regret": 0.015449045225977898, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "tau": 0.3741559563273496 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.3606900164943357, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.3741559563273496 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.45827775488717093, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03407943248748779, + "selected_utility_gain_over_base": -0.07043540477752686, + "selector_regret": 0.056848861277103424, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "tau": 0.3741559563273496 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30129083453678585, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3882270730285144, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18939486145973206, + "selected_utility_gain_over_base": -0.6347416341304779, + "selector_regret": 1.810605138540268, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4763906292645175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1460946798324585, + "selected_utility_gain_over_base": -0.6780418157577515, + "selector_regret": 1.8539053201675415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4720823674110989, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4339821423007203, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.47673474284532225, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.48888892485386093, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06870591640472412, + "selected_utility_gain_over_base": -0.056061550974845886, + "selector_regret": 0.029213324189186096, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "tau": 0.3741559563273496 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3755843205364714, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11081808805465698, + "selected_utility_gain_over_base": -0.013949379324913025, + "selector_regret": 0.019496619701385498, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "tau": 0.3741559563273496 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3785435132544391, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09652663767337799, + "selected_utility_gain_over_base": -0.028240829706192017, + "selector_regret": 1.903473362326622, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3460545899296898, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2999793737204269, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35054441428919914, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2989164896631082, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.35433654964210193, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3440215189574023, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2864812386218567, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3060895394094739, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27674986209929975, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2781929266795631, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3755843205364714, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6900332570075989, + "selected_utility_gain_over_base": -0.03357207775115967, + "selector_regret": 1.3099667429924011, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34895677424461957, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27574476828623157, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2963268387066977, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2728649751609838, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3470083918101673, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.3741559563273496 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3532533835187622, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0696808770298958, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9303191229701042, + "tau": 0.3741559563273496 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3744279152208737, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0455262660980225, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9544737339019775, + "tau": 0.3741559563273496 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2680360950974473, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.3741559563273496 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29770620738185305, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.3741559563273496 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.27387772560678275, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.3741559563273496 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2838401268071768, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8090915828943253, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36516876704401185, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.3741559563273496 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.344497387758629, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02432851493358612, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "tau": 0.3741559563273496 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.3018022542600298, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0287093725055456, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.985974382609129, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.08519926853477955, + "tau": 0.3741559563273496 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.4103646234410795, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.048898130655288696, + "selected_utility_gain_over_base": 0.006163140758872032, + "selector_regret": 0.9887764640152454, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.3741559563273496 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.3708292723058839, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.0179431531578302, + "selected_utility_gain_over_base": 0.9752081632614136, + "selector_regret": 0.11800575815141201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4702558631466304, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20163558423519135, + "selected_utility_gain_over_base": -0.3984549790620804, + "selector_regret": 0.00992196798324585, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.3741559563273496 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3755843205364714, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21116642653942108, + "selected_utility_gain_over_base": -0.38892413675785065, + "selector_regret": 0.3764408975839615, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.3741559563273496 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32005010012998786, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "tau": 0.3741559563273496 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28607089947933784, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3570017249937056, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37134486499196434, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5804248011896158, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5938315939300237, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02717691659927368, + "selected_utility_gain_over_base": -0.008975863456726074, + "selector_regret": 1.9728230834007263, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5137779251382697, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4581486946189829, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21190984547138214, + "selected_utility_gain_over_base": -0.4880453199148178, + "selector_regret": 0.30289267003536224, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "tau": 0.3741559563273496 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3622665827290821, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.3741559563273496 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3550679344666073, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013073742389678955, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "tau": 0.3741559563273496 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.44036449124726545, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.6689300537109375e-06, + "selected_utility_gain_over_base": 1.4454126358032227e-06, + "selector_regret": 1.9999983310699463, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4239855826799761, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.564621925354004e-06, + "selected_utility_gain_over_base": 1.341104507446289e-06, + "selector_regret": 6.008148193359375e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "tau": 0.3741559563273496 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.329217542757152, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 5.200505256652832e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.3741559563273496 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3029974497765087, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28994114758485556, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2844961787136048, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3621977651439929, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3622665827290821, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2897073440884107, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "tau": 0.3741559563273496 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.4254541339617848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.043898362666368484, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.021180112613365054, + "selected_utility_gain_over_base": 0.016081769892480224, + "selector_regret": 0.02271825005300343, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12241578102111816, + "tau": 0.3741559563273496 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.5266725872996079, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030555277597159147, + "selected_candidate_index": 1.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.011683384887874126, + "selected_utility_gain_over_base": 0.006585042166989297, + "selector_regret": 0.01887189270928502, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.13502710359171033, + "tau": 0.3741559563273496 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.37597652661121267, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03856666944921017, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015375842805951834, + "selected_utility_gain_over_base": 0.010277500085067004, + "selector_regret": 0.023190826643258333, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1270157117396593, + "tau": 0.3741559563273496 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.48545416581659245, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01579594612121582, + "selected_utility_gain_over_base": -0.06733909994363785, + "selector_regret": 0.07246536016464233, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.3741559563273496 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.43111137474421807, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017789265140891075, + "selected_utility_gain_over_base": -0.0653457809239626, + "selector_regret": 0.04962191917002201, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.3741559563273496 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.42062384206560416, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0756516233086586, + "selected_utility_gain_over_base": -0.007483422756195068, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.3741559563273496 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2996766962199408, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.3741559563273496 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32254979421285396, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0021464377641677856, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "tau": 0.3741559563273496 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31908267001192103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.3741559563273496 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2791544623573796, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3059562851528022, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3435013909594266, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2996766962199408, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3954257212909892, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5158707077594902, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3741559563273496 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.3303509116866848, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.3741559563273496 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.365539554390633, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "tau": 0.3741559563273496 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.39450829110578517, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06004788354039192, + "selected_utility_gain_over_base": -1.939952116459608, + "selector_regret": 0.06505399569869041, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.3741559563273496 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.3273610892800461, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 3.6666666666666665, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09317883538703124, + "selected_utility_gain_over_base": -0.024414313957095146, + "selector_regret": 0.6594814925144116, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.3741559563273496 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.32791893647515086, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 2.6666666666666665, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11759314934412639, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2529101247588794, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.3741559563273496 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.3074438824092352, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.349846323331197, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09884339570999146, + "selected_utility_gain_over_base": -0.01874975363413493, + "selector_regret": 1.2510029276212056, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05815017223358154, + "tau": 0.3741559563273496 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.30689233926456017, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 3.25, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5731993420049548, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.4779601227492094, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.3741559563273496 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "predicted_margin": 0.37594414604343307, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5951075805351138, + "selected_candidate_index": 0.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5766910687088966, + "selected_utility_gain_over_base": 0.003491726703941822, + "selector_regret": 0.026683504693210125, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.5129444571211934, + "tau": 0.3741559563273496 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.35262652983499576, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.6134321661666036, + "selected_candidate_index": 2.5, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5685367146506906, + "selected_utility_gain_over_base": -0.004662627354264259, + "selector_regret": 0.04489545151591301, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.49461987148970366, + "tau": 0.3741559563273496 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..2048d369d7b00c92a9ad44ad48b8d5b32ef888d5 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.26666666666666666, + "execute_generated": 0.26666666666666666, + "fallback_rate": 0.7333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.35719920667608923, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 1.2244940986235937, + "selected_candidate_index": 3.533333333333333, + "selected_success": 0.2, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8626041044791539, + "selected_utility_gain_over_base": -0.0872664342323939, + "selector_regret": 0.47580571969350177, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.5333333333333333, + "support_gap": 0.7755059013764064, + "tau": 0.3741559563273497 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.23591124007650266, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 3.3333333333333335, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.3741559563273496 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.36, + "execute_generated": 0.36, + "fallback_rate": 0.64, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.44, + "predicted_margin": 0.357845382167158, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 0.9816135169876119, + "selected_candidate_index": 2.4266666666666667, + "selected_success": 0.36, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7684198118560016, + "selected_utility_gain_over_base": -0.006750649367459119, + "selector_regret": 0.3684106833053132, + "success_selector_gap": 0.18666666666666668, + "success_support_gap": 0.09333333333333334, + "support_gap": 0.23527305972451965, + "tau": 0.3741559563273497 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.8888888888888888, + "execute_generated": 0.8888888888888888, + "fallback_rate": 0.1111111111111111, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.7777777777777778, + "predicted_margin": 0.4616574102239464, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.365773618221283, + "selected_candidate_index": 2.5555555555555554, + "selected_success": 0.2777777777777778, + "selected_success_gain_over_base": 0.1111111111111111, + "selected_utility": 0.5970487553212378, + "selected_utility_gain_over_base": 0.2046445765429073, + "selector_regret": 0.7687248629000452, + "success_selector_gap": 0.3888888888888889, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.634226381778717, + "tau": 0.3741559563273497 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.12121212121212122, + "execute_generated": 0.12121212121212122, + "fallback_rate": 0.8787878787878788, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5757575757575758, + "predicted_margin": 0.32668460753400885, + "proposal_oracle_success": 0.45454545454545453, + "proposal_oracle_utility": 1.0546941955884297, + "selected_candidate_index": 4.242424242424242, + "selected_success": 0.2727272727272727, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7366133878628413, + "selected_utility_gain_over_base": -0.03978131666327968, + "selector_regret": 0.32240295003760944, + "success_selector_gap": 0.18181818181818182, + "success_support_gap": 0.5454545454545454, + "support_gap": 0.9453058044115702, + "tau": 0.3741559563273497 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..4a3c9855c9f3b26c201fbae5db4b82f426cecbda --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `rf_regressor` +Selected params: `{"max_depth": 3, "min_samples_leaf": 8, "n_estimators": 128, "n_jobs": 1, "random_state": 0}` +Tau: `0.374156` +Feature set: `basic` +Target: `positive_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4007 | 0.5993 | 0.2780 | 0.3538 | 0.4910 | 0.4838 | 0.2058 | 0.1986 | +| selection | 0.3613 | 0.6387 | 0.2774 | 0.3355 | 0.4387 | 0.5032 | 0.2774 | 0.1677 | +| eval | 0.3542 | 0.6458 | 0.2917 | 0.3056 | 0.4792 | 0.5069 | 0.2917 | 0.2222 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..6f9b0cbd6ad2648ccef33068d3a6c9f8f68b0e85 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.3542 & 0.6458 & 0.2917 & 0.3056 & 0.4792 & 0.5069 & 0.2917 & 0.2222 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3f5019433c608795a7285949ef8878b080ea0ee4 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_positive_envclip_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=rf_regressor +selected_model_params={"max_depth": 3, "min_samples_leaf": 8, "n_estimators": 128, "n_jobs": 1, "random_state": 0} +tau=0.374156 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..d1727cbbba5098502d668e39da43a37d5f8fa886 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test --feature-set basic --target success --k 8 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9d165bd8da151e7cedc253fa071a4cbf452989c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/config.yaml @@ -0,0 +1,13 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: basic +k: 8 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test +seed: 0 +selection_frac: 0.35 +target: success diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e2932d47db7e93e66368f481f66e3fad3b2142c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "06375099001e740a377c402e34b77bf6974be3dc2a468974340ce41eb1b487f0", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "57df43ff4802379dfd60b5209d5a04028d721256406685decededccdbec0d49c", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..f3e7d37f9a11664a633842837a2ebb22ea4c569c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9ad62639d26d017932d1e0228b875cd1fe3cf847 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics.json @@ -0,0 +1,7438 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.4583333333333333, + "fallback_rate": 0.5416666666666666, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5069444444444444, + "proposal_oracle_success": 0.4791666666666667, + "proposal_oracle_utility": 1.054027979976834, + "selected_success": 0.2708333333333333, + "selected_success_gain_over_base": -0.020833333333333332, + "selected_utility": 0.6704846704351777, + "selector_regret": 0.4625271811603818, + "success_selector_gap": 0.25, + "success_support_gap": 0.2916666666666667, + "support_gap": 0.5381004453940679 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates" + ], + "feature_set": "basic", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.37906137184115524, + "fallback_rate": 0.6209386281588448, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.48375451263537905, + "proposal_oracle_success": 0.49097472924187724, + "proposal_oracle_utility": 1.0579715356210844, + "selected_success": 0.33935018050541516, + "selected_success_gain_over_base": 0.061371841155234655, + "selected_utility": 0.7952142179660358, + "selector_regret": 0.3879438841564998, + "success_selector_gap": 0.20938628158844766, + "success_support_gap": 0.20577617328519857, + "support_gap": 0.4109747204182907 + }, + "k": 8, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.4645161290322581, + "fallback_rate": 0.535483870967742, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.3419354838709677, + "selected_success_gain_over_base": 0.06451612903225806, + "selected_utility": 0.808873852979522, + "selector_regret": 0.2741509751334127, + "success_selector_gap": 0.14838709677419354, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "num_fit_candidates": 2216, + "num_fit_rows": 277, + "num_selection_candidates": 1240, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.373318150625259, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4030799603405269, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20728005468845367, + "selected_utility_gain_over_base": -0.39281050860881805, + "selector_regret": 0.0042774975299835205, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3909969586459246, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6859632134437561, + "selected_utility_gain_over_base": -1.314036786556244, + "selector_regret": 1.314036786556244, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19838570058345795, + "selected_utility_gain_over_base": -0.5273159295320511, + "selector_regret": 1.801614299416542, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4063621528211626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "seed": "441046872/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2092108130455017, + "selected_utility_gain_over_base": -0.49074435234069824, + "selector_regret": 0.3055917024612427, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7332244720170006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14637050032615662, + "selected_utility_gain_over_base": -0.4375886023044586, + "selector_regret": 0.036882877349853516, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6386407145727108, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02598150633275509, + "selected_utility_gain_over_base": -0.13570442236959934, + "selector_regret": 1.974018493667245, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.14848772192116344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.3386853712058054, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.00858914852142334, + "seed": "626486813/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006659646867774427, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24483156204223633, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.14087738101788355, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3485165888228928, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.23886865543398522, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07352641969919205, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3025802788019403, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05741874501109123, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.021532438695430756, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1703985370695591, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36221896637920825, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.39277273332080487, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035930830985307693, + "selected_utility_gain_over_base": 0.027663647197186947, + "selector_regret": 0.043276745826005936, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.37670127955549315, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3414075616846885, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.23320400581891038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "seed": "141984615/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08974504470825195, + "selected_utility_gain_over_base": -0.05296391248703003, + "selector_regret": 0.12684154510498047, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.33446532602731266, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005126260221004486, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16255764631774658, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32967755546826816, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.3620474910733139, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3023950264034491, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2410858021363286, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.3612605931849642, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "seed": "1549324805/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06247224286198616, + "selected_utility_gain_over_base": -0.009443063288927078, + "selector_regret": 1.9375277571380138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5148309104053463, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15205490589141846, + "selected_utility_gain_over_base": 0.03498190641403198, + "selector_regret": 1.8479450941085815, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3620474910733139, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "seed": "1324642411/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06546267867088318, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "seed": "1790188787/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09734146296977997, + "selected_utility_gain_over_base": -0.02742600440979004, + "selector_regret": 0.000577777624130249, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4001447846515211, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06716175377368927, + "selected_utility_gain_over_base": -1.9328382462263107, + "selector_regret": 1.9328382462263107, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34339317826396043, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.999999776482582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43218295351476793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29614285515247524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.25921502689335835, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2491980513901653, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40609260924754975, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15893937647342682, + "selected_utility_gain_over_base": -0.6651971191167831, + "selector_regret": 1.8410606235265732, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4266296552350286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.42005283704628976, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12738418579101562, + "selected_utility_gain_over_base": -0.06352423131465912, + "selector_regret": 1.8726158142089844, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40609260924754975, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1854255497455597, + "selected_utility_gain_over_base": -0.04057574272155762, + "selector_regret": 0.6809265911579132, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28595948187094367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36803098356509484, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4266296552350286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4006380153774231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1476449966430664, + "selected_utility_gain_over_base": -1.8523550033569336, + "selector_regret": 1.8523550033569336, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3225177112831876, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "seed": "1073161778/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4395953574467041, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.059747494757175446, + "selected_utility_gain_over_base": -0.5242116078734398, + "selector_regret": 0.010396338999271393, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4043307605631733, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12861795723438263, + "seed": "767747424/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10589394718408585, + "selected_utility_gain_over_base": -0.055791981518268585, + "selector_regret": 0.022724010050296783, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8713820427656174, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3206164719378812, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0079664591467008, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17611322734280455, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13936175405979156, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8606382459402084, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.3214785782650194, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.13204193860292435, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.37854246186477936, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03946288302540779, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.40957871141638863, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05121459439396858, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.020476004108786583, + "selected_utility_gain_over_base": 0.012208820320665836, + "selector_regret": 0.030738590285182, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026529457420110703, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.1270820819653565, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05886904522776604, + "selected_utility_gain_over_base": -0.08383991196751595, + "selector_regret": 1.941130954772234, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.36221896637920825, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5817233097262897, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "seed": "1375492786/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 1.9868215517249155e-08, + "selected_utility_gain_over_base": -1.9999999801317845, + "selector_regret": 0.08251867691675763, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3225177112831876, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.31900666327327665, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018512330949306488, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38381727669892074, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.40682662850481827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05267423391342163, + "selected_utility_gain_over_base": -1.9473257660865784, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3230424852763191, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.3570522478872308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15771149224890263, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5244387767372282, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36264223805353835, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "seed": "1324642411/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0772203803062439, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "seed": "1790188787/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1146477460861206, + "selected_utility_gain_over_base": -0.010119721293449402, + "selector_regret": 0.015666961669921875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04773080348968506, + "selected_utility_gain_over_base": -1.952269196510315, + "selector_regret": 1.952269196510315, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3357431214829802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "seed": "509430392/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 6.142258644104004e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3956988400244218, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20867714285850525, + "selected_utility_gain_over_base": 0.12164306640625, + "selector_regret": 1.7913228571414948, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05141691863536835, + "selected_utility_gain_over_base": -0.0008335858583450317, + "selector_regret": 0.0243864506483078, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3885110983494511, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "seed": "655884605/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11596107482910156, + "selected_utility_gain_over_base": -0.04704560339450836, + "selector_regret": 0.04919204115867615, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1746392846107483, + "selected_utility_gain_over_base": 0.016215860843658447, + "selector_regret": 0.006292939186096191, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36178751330826875, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18482810258865356, + "selected_utility_gain_over_base": -0.006080314517021179, + "selector_regret": 0.02812907099723816, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "seed": "1151410346/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8127282857894897, + "selected_utility_gain_over_base": 0.5867269933223724, + "selector_regret": 0.0011636614799499512, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125, + "selected_utility_gain_over_base": -1.1875, + "selector_regret": 1.1875, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3804379751300867, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "seed": "307068879/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22035211324691772, + "selected_utility_gain_over_base": -0.379738450050354, + "selector_regret": 0.006926536560058594, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3330356097435338, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "seed": "608016290/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "seed": "441046872/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7130289077758789, + "selected_utility_gain_over_base": 0.013073742389678955, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "task_id": "LiftPegUpright-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4163633817877295, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09545384347438812, + "selected_utility_gain_over_base": -0.4885052591562271, + "selector_regret": 0.0539192259311676, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.47855082301126006, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15880563164070036, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18884079158306122, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.027154862880706787, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8111592084169388, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "seed": "1461069713/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005800329148769379, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.42558406978031316, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.026186764240264893, + "seed": "626486813/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.017679253593087196, + "selected_utility_gain_over_base": 0.015749751939438283, + "selector_regret": 0.008507510647177696, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22723394632339478, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.07195363244348905, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09105253219604492, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.908947467803955, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37314194923808613, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.16741520138033664, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07698849588632584, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.1418728306889534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.3804379751300867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39716629921595253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.16741520138033664, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34003810503178633, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "seed": "141984615/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030228236690163612, + "selected_utility_gain_over_base": -0.07260601036250591, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07121429592370987, + "selected_utility_gain_over_base": -0.07149466127157211, + "selector_regret": 1.9287857040762901, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.373318150625259, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.373318150625259, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04602654650807381, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15108097717165947, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1270820819653565, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39493036586117725, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04917396977543831, + "selected_utility_gain_over_base": -1.9508260302245617, + "selector_regret": 1.9508260302245617, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.23320400581891038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04953896999359131, + "seed": "750448296/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17445051670074463, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.46914375133468683, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.31915066590256386, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.43218295351476793, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "seed": "1549324805/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03745478391647339, + "selected_utility_gain_over_base": -0.06706005334854126, + "selector_regret": 0.05347350984811783, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0774010419845581, + "selected_utility_gain_over_base": 0.005485735833644867, + "selector_regret": 1.922598958015442, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3742224732653919, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3230424852763191, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1341685652732849, + "selected_utility_gain_over_base": 0.009401097893714905, + "selector_regret": 1.865831434726715, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40957871141638863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 5.200505256652832e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43218295351476793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3025802788019403, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28595948187094367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3975936184766528, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13359522819519043, + "selected_utility_gain_over_base": -0.6905412673950195, + "selector_regret": 1.8664047718048096, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "seed": "1509980664/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4041207910395072, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "seed": "211245842/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21523693203926086, + "selected_utility_gain_over_base": 0.02432851493358612, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "seed": "1151410346/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21912741661071777, + "selected_utility_gain_over_base": -0.006873875856399536, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2491980513901653, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.18793080747127533, + "selected_utility_gain_over_base": -1.8120691925287247, + "selector_regret": 1.8120691925287247, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1785425841808319, + "selected_utility_gain_over_base": -1.821457415819168, + "selector_regret": 1.821457415819168, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.38358192370540556, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 40, + "max_leaf_nodes": 7, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.4645161290322581, + "fallback_rate": 0.535483870967742, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.3419354838709677, + "selected_success_gain_over_base": 0.06451612903225806, + "selected_utility": 0.808873852979522, + "selector_regret": 0.2741509751334127, + "success_selector_gap": 0.14838709677419354, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5701388888888889, + "low": 0.40138888888888885, + "mean": 0.4888888888888889, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4888888888888889, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 0.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 1.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8206060606060607, + "low": 0.43644444444444447, + "mean": 0.6068282828282828, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6068282828282828, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 1.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.6111111111111112, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.5416666666666666, + "low": 0.3819444444444444, + "mean": 0.4583333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5701388888888889, + "low": 0.40138888888888885, + "mean": 0.4888888888888889, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4888888888888889, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 0.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 1.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 1.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 0.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8206060606060607, + "low": 0.43644444444444447, + "mean": 0.6068282828282828, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6068282828282828, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 1.0, + "PullCube-v1": 0.32, + "PushCube-v1": 0.6111111111111112, + "StackCube-v1": 0.6363636363636364 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.5416666666666666, + "low": 0.3819444444444444, + "mean": 0.4583333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5986111111111112, + "low": 0.4298611111111111, + "mean": 0.5111111111111112, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5111111111111112, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 0.0, + "1113371261/train2": 1.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 0.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 0.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 1.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.5, + "626486813/train1": 0.5, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 0.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.75, + "767747424/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5635555555555556, + "low": 0.17939393939393938, + "mean": 0.39317171717171717, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.39317171717171717, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.68, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.36363636363636365 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.6180555555555556, + "low": 0.4583333333333333, + "mean": 0.5416666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5770833333333333, + "low": 0.40277777777777773, + "mean": 0.48819444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.48819444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6698181818181819, + "low": 0.20315151515151517, + "mean": 0.452040404040404, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.452040404040404, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.44, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.5757575757575758 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.5902777777777778, + "low": 0.4305555555555556, + "mean": 0.5069444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3801811192041846, + "low": 0.3517744368282429, + "mean": 0.36625880830124763, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.36625880830124763, + "per_seed": { + "1073161778/train0": 0.3909969586459246, + "1073161778/train1": 0.3225177112831876, + "1073161778/train2": 0.40035915866759314, + "1113371261/train0": 0.2491980513901653, + "1113371261/train1": 0.3857945730136731, + "1113371261/train2": 0.28595948187094367, + "1151410346/train0": 0.40609260924754975, + "1151410346/train1": 0.40035915866759314, + "1151410346/train2": 0.40035915866759314, + "1261930456/train0": 0.4001447846515211, + "1261930456/train1": 0.3980862785873208, + "1261930456/train2": 0.40957871141638863, + "1324133122/train0": 0.28595948187094367, + "1324133122/train1": 0.30738451498868596, + "1324133122/train2": 0.2491980513901653, + "1324642411/train0": 0.3620474910733139, + "1324642411/train1": 0.32395056402489136, + "1324642411/train2": 0.3230424852763191, + "1375492786/train0": 0.16255764631774658, + "1375492786/train1": 0.5817233097262897, + "1375492786/train2": 0.1270820819653565, + "141984615/train0": 0.23320400581891038, + "141984615/train1": 0.32395056402489136, + "141984615/train2": 0.5202342636576475, + "1461069713/train0": 0.14848772192116344, + "1461069713/train1": 0.3206164719378812, + "1461069713/train2": 0.32395056402489136, + "1480856810/train0": 0.43218295351476793, + "1480856810/train1": 0.3956988400244218, + "1480856810/train2": 0.43218295351476793, + "1509980664/train0": 0.4266296552350286, + "1509980664/train1": 0.36178751330826875, + "1509980664/train2": 0.32395056402489136, + "1522272686/train0": 0.7332244720170006, + "1522272686/train1": 0.4395953574467041, + "1522272686/train2": 0.4163633817877295, + "1549324805/train0": 0.3612605931849642, + "1549324805/train1": 0.3570522478872308, + "1549324805/train2": 0.43218295351476793, + "1578990937/train0": 0.40609260924754975, + "1578990937/train1": 0.40035915866759314, + "1578990937/train2": 0.3975936184766528, + "1747178030/train0": 0.6386407145727108, + "1747178030/train1": 0.4043307605631733, + "1747178030/train2": 0.47855082301126006, + "1790188787/train0": 0.3857945730136731, + "1790188787/train1": 0.38895499821801166, + "1790188787/train2": 0.38895499821801166, + "1822210471/train0": 0.38895499821801166, + "1822210471/train1": 0.15771149224890263, + "1822210471/train2": 0.38895499821801166, + "1845489013/train0": 0.36221896637920825, + "1845489013/train1": 0.38895499821801166, + "1845489013/train2": 0.39716629921595253, + "1999022574/train0": 0.4006380153774231, + "1999022574/train1": 0.38895499821801166, + "1999022574/train2": 0.38895499821801166, + "2012211287/train0": 0.373318150625259, + "2012211287/train1": 0.38358192370540556, + "2012211287/train2": 0.3804379751300867, + "2038594191/train0": 0.32967755546826816, + "2038594191/train1": 0.3225177112831876, + "2038594191/train2": 0.39493036586117725, + "2054899126/train0": 0.22163620371066634, + "2054899126/train1": 0.27996525202086264, + "2054899126/train2": 0.2705486918890879, + "2061635888/train0": 0.29614285515247524, + "2061635888/train1": 0.3857945730136731, + "2061635888/train2": 0.3025802788019403, + "211245842/train0": 0.42005283704628976, + "211245842/train1": 0.38895499821801166, + "211245842/train2": 0.4041207910395072, + "278008524/train0": 0.31326542141341585, + "278008524/train1": 0.3507924735705492, + "278008524/train2": 0.352194269577489, + "307068879/train0": 0.4030799603405269, + "307068879/train1": 0.38358192370540556, + "307068879/train2": 0.3857945730136731, + "412102563/train0": 0.4266296552350286, + "412102563/train1": 0.38895499821801166, + "412102563/train2": 0.38895499821801166, + "440577711/train0": 0.3980862785873208, + "440577711/train1": 0.36264223805353835, + "440577711/train2": 0.3980862785873208, + "441046872/train0": 0.4063621528211626, + "441046872/train1": 0.38358192370540556, + "441046872/train2": 0.38895499821801166, + "509430392/train0": 0.34339317826396043, + "509430392/train1": 0.3357431214829802, + "509430392/train2": 0.38358192370540556, + "585904130/train0": 0.3414075616846885, + "585904130/train1": 0.30738451498868596, + "585904130/train2": 0.34003810503178633, + "608016290/train0": 0.40035915866759314, + "608016290/train1": 0.38358192370540556, + "608016290/train2": 0.3330356097435338, + "626486813/train0": 0.3657290522633051, + "626486813/train1": 0.3584816132025373, + "626486813/train2": 0.40458299674285936, + "636680137/train0": 0.33446532602731266, + "636680137/train1": 0.36221896637920825, + "636680137/train2": 0.373318150625259, + "655884605/train0": 0.25921502689335835, + "655884605/train1": 0.3885110983494511, + "655884605/train2": 0.38358192370540556, + "672328262/train0": 0.3485165888228928, + "672328262/train1": 0.32395056402489136, + "672328262/train2": 0.37314194923808613, + "701667486/train0": 0.36803098356509484, + "701667486/train1": 0.38895499821801166, + "701667486/train2": 0.3980862785873208, + "747260437/train0": 0.2410858021363286, + "747260437/train1": 0.40682662850481827, + "747260437/train2": 0.31915066590256386, + "750448296/train0": 0.4657042217121026, + "750448296/train1": 0.45455990122271744, + "750448296/train2": 0.3758869142473165, + "767747424/train0": 0.360797178408953, + "767747424/train1": 0.31904485971917734, + "767747424/train2": 0.21673854625665817 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.46715076061691896, + "low": 0.3574303741059603, + "mean": 0.4006406978839224, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4006406978839224, + "per_task": { + "LiftPegUpright-v1": 0.38130294013187815, + "PickCube-v1": 0.5297277370838114, + "PullCube-v1": 0.3456744577129766, + "PushCube-v1": 0.3812307027896631, + "StackCube-v1": 0.3652676517012828 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.3773178739953046, + "low": 0.3464877117336371, + "mean": 0.3621548388754108, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5798611111111112, + "low": 0.4131944444444445, + "mean": 0.4972222222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4972222222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5842424242424242, + "low": 0.18424242424242424, + "mean": 0.41090909090909095, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.41090909090909095, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.45454545454545453 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.5555555555555556, + "low": 0.3958333333333333, + "mean": 0.4791666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2489316311703687, + "low": 0.9516223322328491, + "mean": 1.0964438248309307, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.0964438248309307, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 0.7071996331214905, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 0.8138919472694397, + "1151410346/train2": 0.21912741661071777, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.17193320393562317, + "1324642411/train1": 0.1836909055709839, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.08251869678497314, + "1375492786/train2": 2.0, + "141984615/train0": 0.007291515823453665, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.030228236690163612, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09970339387655258, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.8162217140197754, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.0415392704308033, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.09092829376459122, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.09791924059391022, + "1790188787/train1": 0.13031470775604248, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 1.0696808770298958, + "2054899126/train2": 1.0455262660980225, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.21523693203926086, + "278008524/train0": 1.0287093725055456, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.22727864980697632, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.5148025155067444, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7130289077758789, + "509430392/train0": 2.0, + "509430392/train1": 6.164610385894775e-05, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.6825780272483826, + "626486813/train0": 0.043898362666368484, + "626486813/train1": 0.030555277597159147, + "626486813/train2": 0.03856666944921017, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.1651531159877777, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.05267423391342163, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.349846323331197, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 0.5951075805351138, + "767747424/train2": 0.6134321661666036 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2413418491681416, + "low": 0.5217755792898437, + "mean": 0.9521664379400512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9521664379400512, + "per_task": { + "LiftPegUpright-v1": 1.2244940986235937, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 0.9816135169876119, + "PushCube-v1": 1.365773618221283, + "StackCube-v1": 1.0546941955884297 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.196670868741219, + "low": 0.9012325454645583, + "mean": 1.054027979976834, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 2.3666666666666667, + "low": 1.56875, + "mean": 1.9631944444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.9631944444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 7.0, + "1151410346/train0": 1.0, + "1151410346/train1": 3.0, + "1151410346/train2": 0.0, + "1261930456/train0": 3.0, + "1261930456/train1": 1.0, + "1261930456/train2": 6.0, + "1324133122/train0": 6.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 2.0, + "1324642411/train2": 4.0, + "1375492786/train0": 0.0, + "1375492786/train1": 3.0, + "1375492786/train2": 7.0, + "141984615/train0": 7.0, + "141984615/train1": 1.0, + "141984615/train2": 7.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 3.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 7.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 6.0, + "1522272686/train1": 3.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 5.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 6.0, + "1747178030/train2": 5.0, + "1790188787/train0": 7.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 4.0, + "1822210471/train1": 7.0, + "1822210471/train2": 3.0, + "1845489013/train0": 0.0, + "1845489013/train1": 2.0, + "1845489013/train2": 5.0, + "1999022574/train0": 0.0, + "1999022574/train1": 5.0, + "1999022574/train2": 3.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 2.0, + "2038594191/train0": 6.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 6.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 2.0, + "278008524/train0": 0.0, + "278008524/train1": 6.5, + "278008524/train2": 2.0, + "307068879/train0": 1.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 4.0, + "412102563/train2": 2.0, + "440577711/train0": 0.0, + "440577711/train1": 5.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 5.0, + "509430392/train2": 1.0, + "585904130/train0": 6.0, + "585904130/train1": 0.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 7.0, + "626486813/train0": 2.0, + "626486813/train1": 3.0, + "626486813/train2": 0.5, + "636680137/train0": 6.0, + "636680137/train1": 1.0, + "636680137/train2": 5.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 3.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 4.0, + "750448296/train1": 2.3333333333333335, + "750448296/train2": 2.0, + "767747424/train0": 3.25, + "767747424/train1": 1.0, + "767747424/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.968484848484848, + "low": 1.3022222222222222, + "mean": 2.0729292929292926, + "n": 5, + "num_samples": 1000 + }, + "mean": 2.0729292929292926, + "per_task": { + "LiftPegUpright-v1": 0.8, + "PickCube-v1": 3.6666666666666665, + "PullCube-v1": 2.2666666666666666, + "PushCube-v1": 2.0555555555555554, + "StackCube-v1": 1.5757575757575757 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 2.3472222222222223, + "low": 1.5555555555555556, + "mean": 1.9583333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.35694444444444445, + "low": 0.20347222222222222, + "mean": 0.27569444444444446, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.27569444444444446, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 0.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.28888888888888886, + "low": 0.08222222222222222, + "mean": 0.19131313131313132, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19131313131313132, + "per_task": { + "LiftPegUpright-v1": 0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3402777777777778, + "low": 0.2013888888888889, + "mean": 0.2708333333333333, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.03611111111111111, + "low": -0.09166666666666666, + "mean": -0.030555555555555555, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.030555555555555555, + "per_seed": { + "1073161778/train0": -1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": -1.0, + "1261930456/train1": -1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": -1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 1.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": -1.0, + "1999022574/train1": -1.0, + "1999022574/train2": -1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": -1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": -1.0, + "440577711/train0": 1.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": -1.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.05333333333333333, + "low": -0.05624242424242424, + "mean": -0.008565656565656568, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.008565656565656568, + "per_task": { + "LiftPegUpright-v1": -0.06666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": -0.02666666666666667, + "PushCube-v1": 0.1111111111111111, + "StackCube-v1": -0.06060606060606061 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.034722222222222224, + "low": -0.0763888888888889, + "mean": -0.020833333333333332, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8478692709361592, + "low": 0.553924603743427, + "mean": 0.6959834206923714, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6959834206923714, + "per_seed": { + "1073161778/train0": 0.6859632134437561, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.1746392846107483, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.1854255497455597, + "1151410346/train1": 0.8127282857894897, + "1151410346/train2": 0.21912741661071777, + "1261930456/train0": 0.06716175377368927, + "1261930456/train1": 0.04773080348968506, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 1.9868215517249155e-08, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.030228236690163612, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 2.0, + "1480856810/train1": 0.20867714285850525, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.14637050032615662, + "1522272686/train1": 0.059747494757175446, + "1522272686/train2": 0.09545384347438812, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.03745478391647339, + "1578990937/train0": 0.15893937647342682, + "1578990937/train1": 2.0, + "1578990937/train2": 0.13359522819519043, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.09734146296977997, + "1790188787/train1": 0.1146477460861206, + "1790188787/train2": 0.1341685652732849, + "1822210471/train0": 0.06247224286198616, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.0774010419845581, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 0.1476449966430664, + "1999022574/train1": 0.8125, + "1999022574/train2": 0.1785425841808319, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 0.04917396977543831, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05141691863536835, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.12738418579101562, + "211245842/train1": 0.18482810258865356, + "211245842/train2": 0.21523693203926086, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.20728005468845367, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.22035211324691772, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 0.18793080747127533, + "440577711/train0": 2.0, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 2.0, + "441046872/train0": 0.2092108130455017, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.7130289077758789, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.19838570058345795, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.018930166319478303, + "626486813/train1": 0.011202752881217748, + "626486813/train2": 0.012973218690603971, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.11596107482910156, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 0.05267423391342163, + "747260437/train2": 2.0, + "750448296/train0": 0.1115991473197937, + "750448296/train1": 0.717288845529159, + "750448296/train2": 0.09376159558693568, + "767747424/train0": 0.539273236412555, + "767747424/train1": 0.5592513466253877, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7118820036109537, + "low": 0.3259369394835085, + "mean": 0.5580968488727442, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5580968488727442, + "per_task": { + "LiftPegUpright-v1": 0.7437657197316487, + "PickCube-v1": 0.10052394618590672, + "PullCube-v1": 0.7155966088124034, + "PushCube-v1": 0.6125162500474188, + "StackCube-v1": 0.6180817195863435 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8113493838129755, + "low": 0.5411123967917066, + "mean": 0.6704846704351777, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.025342120698607883, + "low": -0.21013489844012326, + "mean": -0.09413464160542288, + "n": 120, + "num_samples": 1000 + }, + "mean": -0.09413464160542288, + "per_seed": { + "1073161778/train0": -1.314036786556244, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.016215860843658447, + "1113371261/train2": 0.0, + "1151410346/train0": -0.04057574272155762, + "1151410346/train1": 0.5867269933223724, + "1151410346/train2": -0.006873875856399536, + "1261930456/train0": -1.9328382462263107, + "1261930456/train1": -1.952269196510315, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": -1.9999999801317845, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": -0.07260601036250591, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 0.12164306640625, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": -0.4375886023044586, + "1522272686/train1": -0.5242116078734398, + "1522272686/train2": -0.4885052591562271, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": -0.06706005334854126, + "1578990937/train0": -0.6651971191167831, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": -0.6905412673950195, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": -0.02742600440979004, + "1790188787/train1": -0.010119721293449402, + "1790188787/train2": 0.009401097893714905, + "1822210471/train0": -0.009443063288927078, + "1822210471/train1": 0.0, + "1822210471/train2": 0.005485735833644867, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": -1.8523550033569336, + "1999022574/train1": -1.1875, + "1999022574/train2": -1.821457415819168, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": -1.9508260302245617, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": -0.0008335858583450317, + "2061635888/train2": 0.0, + "211245842/train0": -0.06352423131465912, + "211245842/train1": -0.006080314517021179, + "211245842/train2": 0.02432851493358612, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": -0.39281050860881805, + "307068879/train1": 0.0, + "307068879/train2": -0.379738450050354, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": -1.8120691925287247, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 0.0, + "440577711/train2": 1.9638472199440002, + "441046872/train0": -0.49074435234069824, + "441046872/train1": 0.0, + "441046872/train2": 0.013073742389678955, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": -0.5273159295320511, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.013831823598593473, + "626486813/train1": 0.006104410160332918, + "626486813/train2": 0.007874875969719142, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": -0.04704560339450836, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": -1.9473257660865784, + "747260437/train2": 0.0, + "750448296/train0": -0.005994002024332683, + "750448296/train1": 0.5996956961850325, + "750448296/train2": -0.023831553757190704, + "767747424/train0": -0.033926105592399836, + "767747424/train1": -0.013947995379567146, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.06874204878554199, + "low": -0.3431968281450164, + "mean": -0.13746294830127087, + "n": 5, + "num_samples": 1000 + }, + "mean": -0.13746294830127087, + "per_task": { + "LiftPegUpright-v1": -0.2061048189798991, + "PickCube-v1": -0.4834351564447085, + "PullCube-v1": -0.05957385241105738, + "PushCube-v1": 0.22011207126908833, + "StackCube-v1": -0.1583129849397775 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.03229221885542049, + "low": -0.1732966163499104, + "mean": -0.0713349160071596, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6053354703098497, + "low": 0.35603929869361717, + "mean": 0.47097727502857906, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.47097727502857906, + "per_seed": { + "1073161778/train0": 1.314036786556244, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.006292939186096191, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6809265911579132, + "1151410346/train1": 0.0011636614799499512, + "1151410346/train2": 0.0, + "1261930456/train0": 1.9328382462263107, + "1261930456/train1": 1.952269196510315, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.06546267867088318, + "1324642411/train1": 0.0772203803062439, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.08251867691675763, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.005800329148769379, + "1480856810/train0": 0.0, + "1480856810/train1": 1.7913228571414948, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 0.0, + "1522272686/train0": 0.036882877349853516, + "1522272686/train1": 0.010396338999271393, + "1522272686/train2": 0.0539192259311676, + "1549324805/train0": 0.0, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.05347350984811783, + "1578990937/train0": 1.8410606235265732, + "1578990937/train1": 0.0, + "1578990937/train2": 1.8664047718048096, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.000577777624130249, + "1790188787/train1": 0.015666961669921875, + "1790188787/train2": 1.865831434726715, + "1822210471/train0": 1.9375277571380138, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.922598958015442, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.8523550033569336, + "1999022574/train1": 1.1875, + "1999022574/train2": 1.821457415819168, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2763946652412415, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.9508260302245617, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.0243864506483078, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8726158142089844, + "211245842/train1": 0.02812907099723816, + "211245842/train2": 0.0, + "278008524/train0": 0.985974382609129, + "278008524/train1": 0.9949396047741175, + "278008524/train2": 1.0932139214128256, + "307068879/train0": 0.0042774975299835205, + "307068879/train1": 0.0, + "307068879/train2": 0.006926536560058594, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.8120691925287247, + "440577711/train0": 0.0, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 0.0, + "441046872/train0": 0.3055917024612427, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.0, + "509430392/train0": 1.999999776482582, + "509430392/train1": 6.142258644104004e-05, + "509430392/train2": 5.200505256652832e-06, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.801614299416542, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.02496819634689018, + "626486813/train1": 0.0193525247159414, + "626486813/train2": 0.025593450758606195, + "636680137/train0": 0.005126260221004486, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.04919204115867615, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6582622130711874, + "750448296/train1": 0.6532144285738468, + "750448296/train2": 1.2705709015329678, + "767747424/train0": 0.5118862283416092, + "767747424/train1": 0.03869148716330528, + "767747424/train2": 0.04225692339241505 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6196751023051377, + "low": 0.2139395446117139, + "mean": 0.4392431564182607, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.4392431564182607, + "per_task": { + "LiftPegUpright-v1": 0.5706221928199132, + "PickCube-v1": 0.03373281409343084, + "PullCube-v1": 0.397877087958364, + "PushCube-v1": 0.7532573681738641, + "StackCube-v1": 0.4407263190457315 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.5858757785624928, + "low": 0.34323343148355856, + "mean": 0.4625271811603818, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3326388888888889, + "low": 0.19305555555555556, + "mean": 0.25902777777777775, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.25902777777777775, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 0.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3618181818181818, + "low": 0.11777777777777779, + "mean": 0.24626262626262627, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.24626262626262627, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.2, + "PushCube-v1": 0.3888888888888889, + "StackCube-v1": 0.24242424242424243 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.3194444444444444, + "low": 0.18055555555555555, + "mean": 0.25, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39861111111111114, + "low": 0.2375, + "mean": 0.3194444444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3194444444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7757575757575758, + "low": 0.23733333333333334, + "mean": 0.501090909090909, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.501090909090909, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.09333333333333334, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.5454545454545454 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7096626987680793, + "low": 0.43689682670713714, + "mean": 0.5784295155162302, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5784295155162302, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.2928003668785095, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 1.1861080527305603, + "1151410346/train2": 1.7808725833892822, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8280667960643768, + "1324642411/train1": 1.8163090944290161, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.9174813032150269, + "1375492786/train2": 0.0, + "141984615/train0": 0.27848575497046113, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2555490341037512, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.059639059007167816, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.1837782859802246, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.255916815251112, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.20652779191732407, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.9020807594060898, + "1790188787/train1": 1.8696852922439575, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.9303191229701042, + "2054899126/train2": 0.9544737339019775, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7847630679607391, + "278008524/train0": 0.08519926853477955, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7727213501930237, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.4851974844932556, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.286971092224121, + "509430392/train0": 0.0, + "509430392/train1": 1.999938353896141, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.3174219727516174, + "626486813/train0": 0.12241578102111816, + "626486813/train1": 0.13502710359171033, + "626486813/train2": 0.1270157117396593, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.8348468840122223, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.22598901391029358, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05815017223358154, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.5129444571211934, + "767747424/train2": 0.49461987148970366 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.4353523810704547, + "low": 0.47464505295703807, + "mean": 0.8912108774023751, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8912108774023751, + "per_task": { + "LiftPegUpright-v1": 0.7755059013764064, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.23527305972451965, + "PushCube-v1": 0.634226381778717, + "StackCube-v1": 0.9453058044115702 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.6574370106811531, + "low": 0.4129596278103741, + "mean": 0.5381004453940679, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3835819237054062, + "low": 0.3835819237054062, + "mean": 0.3835819237054062, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3835819237054062, + "per_seed": { + "1073161778/train0": 0.38358192370540556, + "1073161778/train1": 0.38358192370540556, + "1073161778/train2": 0.38358192370540556, + "1113371261/train0": 0.38358192370540556, + "1113371261/train1": 0.38358192370540556, + "1113371261/train2": 0.38358192370540556, + "1151410346/train0": 0.38358192370540556, + "1151410346/train1": 0.38358192370540556, + "1151410346/train2": 0.38358192370540556, + "1261930456/train0": 0.38358192370540556, + "1261930456/train1": 0.38358192370540556, + "1261930456/train2": 0.38358192370540556, + "1324133122/train0": 0.38358192370540556, + "1324133122/train1": 0.38358192370540556, + "1324133122/train2": 0.38358192370540556, + "1324642411/train0": 0.38358192370540556, + "1324642411/train1": 0.38358192370540556, + "1324642411/train2": 0.38358192370540556, + "1375492786/train0": 0.38358192370540556, + "1375492786/train1": 0.38358192370540556, + "1375492786/train2": 0.38358192370540556, + "141984615/train0": 0.38358192370540556, + "141984615/train1": 0.38358192370540556, + "141984615/train2": 0.38358192370540556, + "1461069713/train0": 0.38358192370540556, + "1461069713/train1": 0.38358192370540556, + "1461069713/train2": 0.38358192370540556, + "1480856810/train0": 0.38358192370540556, + "1480856810/train1": 0.38358192370540556, + "1480856810/train2": 0.38358192370540556, + "1509980664/train0": 0.38358192370540556, + "1509980664/train1": 0.38358192370540556, + "1509980664/train2": 0.38358192370540556, + "1522272686/train0": 0.38358192370540556, + "1522272686/train1": 0.38358192370540556, + "1522272686/train2": 0.38358192370540556, + "1549324805/train0": 0.38358192370540556, + "1549324805/train1": 0.38358192370540556, + "1549324805/train2": 0.38358192370540556, + "1578990937/train0": 0.38358192370540556, + "1578990937/train1": 0.38358192370540556, + "1578990937/train2": 0.38358192370540556, + "1747178030/train0": 0.38358192370540556, + "1747178030/train1": 0.38358192370540556, + "1747178030/train2": 0.38358192370540556, + "1790188787/train0": 0.38358192370540556, + "1790188787/train1": 0.38358192370540556, + "1790188787/train2": 0.38358192370540556, + "1822210471/train0": 0.38358192370540556, + "1822210471/train1": 0.38358192370540556, + "1822210471/train2": 0.38358192370540556, + "1845489013/train0": 0.38358192370540556, + "1845489013/train1": 0.38358192370540556, + "1845489013/train2": 0.38358192370540556, + "1999022574/train0": 0.38358192370540556, + "1999022574/train1": 0.38358192370540556, + "1999022574/train2": 0.38358192370540556, + "2012211287/train0": 0.38358192370540556, + "2012211287/train1": 0.38358192370540556, + "2012211287/train2": 0.38358192370540556, + "2038594191/train0": 0.38358192370540556, + "2038594191/train1": 0.38358192370540556, + "2038594191/train2": 0.38358192370540556, + "2054899126/train0": 0.38358192370540556, + "2054899126/train1": 0.38358192370540556, + "2054899126/train2": 0.38358192370540556, + "2061635888/train0": 0.38358192370540556, + "2061635888/train1": 0.38358192370540556, + "2061635888/train2": 0.38358192370540556, + "211245842/train0": 0.38358192370540556, + "211245842/train1": 0.38358192370540556, + "211245842/train2": 0.38358192370540556, + "278008524/train0": 0.38358192370540556, + "278008524/train1": 0.38358192370540556, + "278008524/train2": 0.38358192370540556, + "307068879/train0": 0.38358192370540556, + "307068879/train1": 0.38358192370540556, + "307068879/train2": 0.38358192370540556, + "412102563/train0": 0.38358192370540556, + "412102563/train1": 0.38358192370540556, + "412102563/train2": 0.38358192370540556, + "440577711/train0": 0.38358192370540556, + "440577711/train1": 0.38358192370540556, + "440577711/train2": 0.38358192370540556, + "441046872/train0": 0.38358192370540556, + "441046872/train1": 0.38358192370540556, + "441046872/train2": 0.38358192370540556, + "509430392/train0": 0.38358192370540556, + "509430392/train1": 0.38358192370540556, + "509430392/train2": 0.38358192370540556, + "585904130/train0": 0.38358192370540556, + "585904130/train1": 0.38358192370540556, + "585904130/train2": 0.38358192370540556, + "608016290/train0": 0.38358192370540556, + "608016290/train1": 0.38358192370540556, + "608016290/train2": 0.38358192370540556, + "626486813/train0": 0.38358192370540556, + "626486813/train1": 0.38358192370540556, + "626486813/train2": 0.38358192370540556, + "636680137/train0": 0.38358192370540556, + "636680137/train1": 0.38358192370540556, + "636680137/train2": 0.38358192370540556, + "655884605/train0": 0.38358192370540556, + "655884605/train1": 0.38358192370540556, + "655884605/train2": 0.38358192370540556, + "672328262/train0": 0.38358192370540556, + "672328262/train1": 0.38358192370540556, + "672328262/train2": 0.38358192370540556, + "701667486/train0": 0.38358192370540556, + "701667486/train1": 0.38358192370540556, + "701667486/train2": 0.38358192370540556, + "747260437/train0": 0.38358192370540556, + "747260437/train1": 0.38358192370540556, + "747260437/train2": 0.38358192370540556, + "750448296/train0": 0.3835819237054056, + "750448296/train1": 0.3835819237054056, + "750448296/train2": 0.3835819237054056, + "767747424/train0": 0.38358192370540556, + "767747424/train1": 0.38358192370540556, + "767747424/train2": 0.38358192370540556 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3835819237054058, + "low": 0.38358192370540556, + "mean": 0.38358192370540567, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.38358192370540567, + "per_task": { + "LiftPegUpright-v1": 0.3835819237054056, + "PickCube-v1": 0.3835819237054056, + "PullCube-v1": 0.38358192370540595, + "PushCube-v1": 0.38358192370540567, + "StackCube-v1": 0.38358192370540545 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.38358192370540634, + "low": 0.38358192370540634, + "mean": 0.38358192370540634, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "success", + "tau": 0.38358192370540556 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..5f0e63d5ff679d652a19e8ef5dcd2a07830cf2c1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3909969586459246, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6859632134437561, + "selected_utility_gain_over_base": -1.314036786556244, + "selector_regret": 1.314036786556244, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3225177112831876, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "tau": 0.38358192370540556 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2491980513901653, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.38358192370540556 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1746392846107483, + "selected_utility_gain_over_base": 0.016215860843658447, + "selector_regret": 0.006292939186096191, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.38358192370540556 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28595948187094367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.38358192370540556 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40609260924754975, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1854255497455597, + "selected_utility_gain_over_base": -0.04057574272155762, + "selector_regret": 0.6809265911579132, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.38358192370540556 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8127282857894897, + "selected_utility_gain_over_base": 0.5867269933223724, + "selector_regret": 0.0011636614799499512, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "tau": 0.38358192370540556 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21912741661071777, + "selected_utility_gain_over_base": -0.006873875856399536, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "tau": 0.38358192370540556 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4001447846515211, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06716175377368927, + "selected_utility_gain_over_base": -1.9328382462263107, + "selector_regret": 1.9328382462263107, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04773080348968506, + "selected_utility_gain_over_base": -1.952269196510315, + "selector_regret": 1.952269196510315, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40957871141638863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.28595948187094367, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.38358192370540556 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.38358192370540556 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2491980513901653, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.38358192370540556 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3620474910733139, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06546267867088318, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "tau": 0.38358192370540556 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0772203803062439, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "tau": 0.38358192370540556 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3230424852763191, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16255764631774658, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5817233097262897, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 1.9868215517249155e-08, + "selected_utility_gain_over_base": -1.9999999801317845, + "selector_regret": 0.08251867691675763, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "tau": 0.38358192370540556 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1270820819653565, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.23320400581891038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "tau": 0.38358192370540556 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.38358192370540556 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.5202342636576475, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.030228236690163612, + "selected_utility_gain_over_base": -0.07260601036250591, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "tau": 0.38358192370540556 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.14848772192116344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.38358192370540556 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.3206164719378812, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.38358192370540556 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005800329148769379, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "tau": 0.38358192370540556 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43218295351476793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3956988400244218, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20867714285850525, + "selected_utility_gain_over_base": 0.12164306640625, + "selector_regret": 1.7913228571414948, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.43218295351476793, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4266296552350286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36178751330826875, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "tau": 0.38358192370540556 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7332244720170006, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14637050032615662, + "selected_utility_gain_over_base": -0.4375886023044586, + "selector_regret": 0.036882877349853516, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.38358192370540556 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4395953574467041, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.059747494757175446, + "selected_utility_gain_over_base": -0.5242116078734398, + "selector_regret": 0.010396338999271393, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.38358192370540556 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4163633817877295, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09545384347438812, + "selected_utility_gain_over_base": -0.4885052591562271, + "selector_regret": 0.0539192259311676, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.38358192370540556 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.3612605931849642, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "tau": 0.38358192370540556 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.3570522478872308, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.38358192370540556 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.43218295351476793, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03745478391647339, + "selected_utility_gain_over_base": -0.06706005334854126, + "selector_regret": 0.05347350984811783, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "tau": 0.38358192370540556 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40609260924754975, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15893937647342682, + "selected_utility_gain_over_base": -0.6651971191167831, + "selector_regret": 1.8410606235265732, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.17586350440979, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3975936184766528, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13359522819519043, + "selected_utility_gain_over_base": -0.6905412673950195, + "selector_regret": 1.8664047718048096, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6386407145727108, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4043307605631733, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.47855082301126006, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09734146296977997, + "selected_utility_gain_over_base": -0.02742600440979004, + "selector_regret": 0.000577777624130249, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "tau": 0.38358192370540556 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1146477460861206, + "selected_utility_gain_over_base": -0.010119721293449402, + "selector_regret": 0.015666961669921875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "tau": 0.38358192370540556 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1341685652732849, + "selected_utility_gain_over_base": 0.009401097893714905, + "selector_regret": 1.865831434726715, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06247224286198616, + "selected_utility_gain_over_base": -0.009443063288927078, + "selector_regret": 1.9375277571380138, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15771149224890263, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0774010419845581, + "selected_utility_gain_over_base": 0.005485735833644867, + "selector_regret": 1.922598958015442, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36221896637920825, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39716629921595253, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4006380153774231, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1476449966430664, + "selected_utility_gain_over_base": -1.8523550033569336, + "selector_regret": 1.8523550033569336, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8125, + "selected_utility_gain_over_base": -1.1875, + "selector_regret": 1.1875, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1785425841808319, + "selected_utility_gain_over_base": -1.821457415819168, + "selector_regret": 1.821457415819168, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.373318150625259, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3804379751300867, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32967755546826816, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3225177112831876, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.39493036586117725, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.04917396977543831, + "selected_utility_gain_over_base": -1.9508260302245617, + "selector_regret": 1.9508260302245617, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22163620371066634, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.38358192370540556 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.27996525202086264, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0696808770298958, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9303191229701042, + "tau": 0.38358192370540556 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2705486918890879, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0455262660980225, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9544737339019775, + "tau": 0.38358192370540556 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.29614285515247524, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.38358192370540556 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05141691863536835, + "selected_utility_gain_over_base": -0.0008335858583450317, + "selector_regret": 0.0243864506483078, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.38358192370540556 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3025802788019403, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.38358192370540556 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.42005283704628976, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12738418579101562, + "selected_utility_gain_over_base": -0.06352423131465912, + "selector_regret": 1.8726158142089844, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18482810258865356, + "selected_utility_gain_over_base": -0.006080314517021179, + "selector_regret": 0.02812907099723816, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.38358192370540556 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4041207910395072, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.21523693203926086, + "selected_utility_gain_over_base": 0.02432851493358612, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "tau": 0.38358192370540556 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.31326542141341585, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0287093725055456, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.985974382609129, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.08519926853477955, + "tau": 0.38358192370540556 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.3507924735705492, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 6.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.9949396047741175, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.38358192370540556 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.352194269577489, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.0932139214128256, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4030799603405269, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20728005468845367, + "selected_utility_gain_over_base": -0.39281050860881805, + "selector_regret": 0.0042774975299835205, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.38358192370540556 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.38358192370540556 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3857945730136731, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.22035211324691772, + "selected_utility_gain_over_base": -0.379738450050354, + "selector_regret": 0.006926536560058594, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "tau": 0.38358192370540556 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4266296552350286, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.18793080747127533, + "selected_utility_gain_over_base": -1.8120691925287247, + "selector_regret": 1.8120691925287247, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.36264223805353835, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.036152780055999756, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9638472199440002, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4063621528211626, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2092108130455017, + "selected_utility_gain_over_base": -0.49074435234069824, + "selector_regret": 0.3055917024612427, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "tau": 0.38358192370540556 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.38358192370540556 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7130289077758789, + "selected_utility_gain_over_base": 0.013073742389678955, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "tau": 0.38358192370540556 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34339317826396043, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.999999776482582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3357431214829802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 6.142258644104004e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "tau": 0.38358192370540556 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 5.200505256652832e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.38358192370540556 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3414075616846885, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.30738451498868596, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34003810503178633, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.40035915866759314, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19838570058345795, + "selected_utility_gain_over_base": -0.5273159295320511, + "selector_regret": 1.801614299416542, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.274298369884491, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3330356097435338, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "tau": 0.38358192370540556 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.3657290522633051, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.043898362666368484, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.018930166319478303, + "selected_utility_gain_over_base": 0.013831823598593473, + "selector_regret": 0.02496819634689018, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12241578102111816, + "tau": 0.38358192370540556 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.3584816132025373, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030555277597159147, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.011202752881217748, + "selected_utility_gain_over_base": 0.006104410160332918, + "selector_regret": 0.0193525247159414, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.13502710359171033, + "tau": 0.38358192370540556 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.40458299674285936, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03856666944921017, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.012973218690603971, + "selected_utility_gain_over_base": 0.007874875969719142, + "selector_regret": 0.025593450758606195, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1270157117396593, + "tau": 0.38358192370540556 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.33446532602731266, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005126260221004486, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.38358192370540556 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.36221896637920825, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.38358192370540556 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.373318150625259, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.38358192370540556 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.25921502689335835, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.38358192370540556 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3885110983494511, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11596107482910156, + "selected_utility_gain_over_base": -0.04704560339450836, + "selector_regret": 0.04919204115867615, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "tau": 0.38358192370540556 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.38358192370540556, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.38358192370540556 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3485165888228928, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32395056402489136, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.37314194923808613, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36803098356509484, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38895499821801166, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3980862785873208, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.38358192370540556 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.2410858021363286, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.38358192370540556 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.40682662850481827, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05267423391342163, + "selected_utility_gain_over_base": -1.9473257660865784, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "tau": 0.38358192370540556 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.31915066590256386, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.38358192370540556 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.4657042217121026, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1115991473197937, + "selected_utility_gain_over_base": -0.005994002024332683, + "selector_regret": 0.6582622130711874, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.3835819237054056 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.45455990122271744, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 2.3333333333333335, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.717288845529159, + "selected_utility_gain_over_base": 0.5996956961850325, + "selector_regret": 0.6532144285738468, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.3835819237054056 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.3758869142473165, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.349846323331197, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09376159558693568, + "selected_utility_gain_over_base": -0.023831553757190704, + "selector_regret": 1.2705709015329678, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05815017223358154, + "tau": 0.3835819237054056 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.360797178408953, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 3.25, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.539273236412555, + "selected_utility_gain_over_base": -0.033926105592399836, + "selector_regret": 0.5118862283416092, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.38358192370540556 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "predicted_margin": 0.31904485971917734, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5951075805351138, + "selected_candidate_index": 1.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5592513466253877, + "selected_utility_gain_over_base": -0.013947995379567146, + "selector_regret": 0.03869148716330528, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.5129444571211934, + "tau": 0.38358192370540556 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.21673854625665817, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.6134321661666036, + "selected_candidate_index": 1.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5731993420049548, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04225692339241505, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.49461987148970366, + "tau": 0.38358192370540556 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..97636cf3e2e58d2c3d6b1593630461d001173e05 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.4666666666666667, + "execute_generated": 0.4666666666666667, + "fallback_rate": 0.5333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.38130294013187815, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 1.2244940986235937, + "selected_candidate_index": 0.8, + "selected_success": 0.13333333333333333, + "selected_success_gain_over_base": -0.06666666666666667, + "selected_utility": 0.7437657197316487, + "selected_utility_gain_over_base": -0.2061048189798991, + "selector_regret": 0.5706221928199132, + "success_selector_gap": 0.4, + "success_support_gap": 0.5333333333333333, + "support_gap": 0.7755059013764064, + "tau": 0.3835819237054056 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5297277370838114, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 3.6666666666666665, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10052394618590672, + "selected_utility_gain_over_base": -0.4834351564447085, + "selector_regret": 0.03373281409343084, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.3835819237054056 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.32, + "execute_generated": 0.32, + "fallback_rate": 0.68, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.44, + "predicted_margin": 0.3456744577129766, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 0.9816135169876119, + "selected_candidate_index": 2.2666666666666666, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": -0.02666666666666667, + "selected_utility": 0.7155966088124034, + "selected_utility_gain_over_base": -0.05957385241105738, + "selector_regret": 0.397877087958364, + "success_selector_gap": 0.2, + "success_support_gap": 0.09333333333333334, + "support_gap": 0.23527305972451965, + "tau": 0.38358192370540595 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.6111111111111112, + "execute_generated": 0.6111111111111112, + "fallback_rate": 0.3888888888888889, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.7777777777777778, + "predicted_margin": 0.3812307027896631, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.365773618221283, + "selected_candidate_index": 2.0555555555555554, + "selected_success": 0.2777777777777778, + "selected_success_gain_over_base": 0.1111111111111111, + "selected_utility": 0.6125162500474188, + "selected_utility_gain_over_base": 0.22011207126908833, + "selector_regret": 0.7532573681738641, + "success_selector_gap": 0.3888888888888889, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.634226381778717, + "tau": 0.38358192370540567 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.6363636363636364, + "execute_generated": 0.6363636363636364, + "fallback_rate": 0.36363636363636365, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5757575757575758, + "predicted_margin": 0.3652676517012828, + "proposal_oracle_success": 0.45454545454545453, + "proposal_oracle_utility": 1.0546941955884297, + "selected_candidate_index": 1.5757575757575757, + "selected_success": 0.21212121212121213, + "selected_success_gain_over_base": -0.06060606060606061, + "selected_utility": 0.6180817195863435, + "selected_utility_gain_over_base": -0.1583129849397775, + "selector_regret": 0.4407263190457315, + "success_selector_gap": 0.24242424242424243, + "success_support_gap": 0.5454545454545454, + "support_gap": 0.9453058044115702, + "tau": 0.38358192370540545 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..a943a9c93140dae500a60e5144d658583bae6054 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 40, "max_leaf_nodes": 7, "random_state": 0}` +Tau: `0.383582` +Feature set: `basic` +Target: `success` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.3791 | 0.6209 | 0.2780 | 0.3394 | 0.4910 | 0.4838 | 0.2058 | 0.2094 | +| selection | 0.4645 | 0.5355 | 0.2774 | 0.3419 | 0.4387 | 0.5032 | 0.2774 | 0.1484 | +| eval | 0.4583 | 0.5417 | 0.2917 | 0.2708 | 0.4792 | 0.5069 | 0.2917 | 0.2500 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..349e43e4da3b8dccbba4a6ba57e364a914bedefe --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.4583 & 0.5417 & 0.2917 & 0.2708 & 0.4792 & 0.5069 & 0.2917 & 0.2500 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8b634241fb817226ae474212a863157950958dfc --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_basic_success_envclip_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 40, "max_leaf_nodes": 7, "random_state": 0} +tau=0.383582 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..8eef255409fd35e08511573a63b6429541a42f7e --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test --feature-set context_tangent --target positive_margin --k 8 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65bfd222352986e5a9d6ba5bd985c6af685f9a81 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/config.yaml @@ -0,0 +1,13 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: context_tangent +k: 8 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test +seed: 0 +selection_frac: 0.35 +target: positive_margin diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e2932d47db7e93e66368f481f66e3fad3b2142c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "06375099001e740a377c402e34b77bf6974be3dc2a468974340ce41eb1b487f0", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "57df43ff4802379dfd60b5209d5a04028d721256406685decededccdbec0d49c", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..f3e7d37f9a11664a633842837a2ebb22ea4c569c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4402d3b29e940090d2f9b39c98cb844e08b297d7 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics.json @@ -0,0 +1,7507 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.375, + "fallback_rate": 0.625, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5069444444444444, + "proposal_oracle_success": 0.4791666666666667, + "proposal_oracle_utility": 1.054027979976834, + "selected_success": 0.2916666666666667, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7356658353665908, + "selector_regret": 0.38589712319258573, + "success_selector_gap": 0.2152777777777778, + "success_support_gap": 0.2916666666666667, + "support_gap": 0.5381004453940679 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "target_task_hash_00", + "target_task_hash_01", + "target_task_hash_02", + "target_task_hash_03", + "target_task_hash_04", + "target_task_hash_05", + "target_task_hash_06", + "target_task_hash_07", + "source_task_hash_00", + "source_task_hash_01", + "source_task_hash_02", + "source_task_hash_03", + "source_task_hash_04", + "source_task_hash_05", + "source_task_hash_06", + "source_task_hash_07", + "instruction_hash_00", + "instruction_hash_01", + "instruction_hash_02", + "instruction_hash_03", + "instruction_hash_04", + "instruction_hash_05", + "instruction_hash_06", + "instruction_hash_07", + "source_target_same_task", + "instruction_chars_per_128", + "instruction_words_per_32", + "tangent_00", + "tangent_01", + "tangent_02", + "tangent_03", + "tangent_04", + "tangent_05", + "tangent_06", + "tangent_07", + "tangent_08", + "tangent_09", + "tangent_10", + "tangent_11", + "tangent_12", + "tangent_13", + "tangent_14", + "tangent_15", + "tangent_16", + "tangent_17", + "tangent_18", + "tangent_19", + "tangent_20", + "abs_tangent_00", + "abs_tangent_01", + "abs_tangent_02", + "abs_tangent_03", + "abs_tangent_04", + "abs_tangent_05", + "abs_tangent_06", + "abs_tangent_07", + "abs_tangent_08", + "abs_tangent_09", + "abs_tangent_10", + "abs_tangent_11", + "abs_tangent_12", + "abs_tangent_13", + "abs_tangent_14", + "abs_tangent_15", + "abs_tangent_16", + "abs_tangent_17", + "abs_tangent_18", + "abs_tangent_19", + "abs_tangent_20" + ], + "feature_set": "context_tangent", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.44765342960288806, + "fallback_rate": 0.5523465703971119, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.48375451263537905, + "proposal_oracle_success": 0.49097472924187724, + "proposal_oracle_utility": 1.0579715356210844, + "selected_success": 0.48375451263537905, + "selected_success_gain_over_base": 0.20577617328519857, + "selected_utility": 1.0619221195790391, + "selector_regret": 0.13691463883434984, + "success_selector_gap": 0.07220216606498195, + "success_support_gap": 0.20577617328519857, + "support_gap": 0.4109747204182907 + }, + "k": 8, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.4064516129032258, + "fallback_rate": 0.5935483870967742, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.36774193548387096, + "selected_success_gain_over_base": 0.09032258064516129, + "selected_utility": 0.869380844928204, + "selector_regret": 0.230353160301644, + "success_selector_gap": 0.12903225806451613, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "num_fit_candidates": 2216, + "num_fit_rows": 277, + "num_selection_candidates": 1240, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4750392109884963, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7064330577850342, + "selected_utility_gain_over_base": -0.017172276973724365, + "selector_regret": 1.2935669422149658, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5443095032644824, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2115575522184372, + "selected_utility_gain_over_base": -0.38853301107883453, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4346372534889307, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6875576972961426, + "selected_utility_gain_over_base": -1.3124423027038574, + "selector_regret": 1.3124423027038574, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48980866861669325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6760407205809709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "seed": "441046872/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5148025155067444, + "selected_utility_gain_over_base": -0.18515264987945557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09979768262139747, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9365813789082288, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2044135569629093, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8383140712976456, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.029854794800442567, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.044758936664508156, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.00858914852142334, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006659646867774427, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24483156204223633, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8730337849053196, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1429876834154129, + "selected_utility_gain_over_base": -1.857012316584587, + "selector_regret": 0.011578619480133057, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.013741725805362232, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.4801439518370624, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05045728012919426, + "selected_utility_gain_over_base": -0.026531215757131577, + "selector_regret": 0.10005763545632362, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.4897843220319693, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05741874501109123, + "seed": "278008524/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03823399543762207, + "selected_utility_gain_over_base": 0.0023476891219615936, + "selector_regret": 0.019184749573469162, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1703985370695591, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32943668699513484, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.2742729016409696, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.011503218154861237, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14570795144295595, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.013890986859197135, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "seed": "141984615/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1488960434386867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.23786709945842635, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005126260221004486, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.02424133054418011, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.12776527911809626, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16493003734275938, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.052877087476571, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.334492767232634, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.09061742718972163, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4252202012461091, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10624494403600693, + "selected_utility_gain_over_base": 0.056661270558834076, + "selector_regret": 1.893755055963993, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.039192806066983506, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "seed": "1549324805/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26476739579345127, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18188511768475743, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6446552646547107, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2039301366121156, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "seed": "1324642411/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06546267867088318, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.610448511026168, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "seed": "1790188787/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06047999858856201, + "selected_utility_gain_over_base": -0.064287468791008, + "selector_regret": 0.037439242005348206, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6177992215925546, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2460083281337516, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.999999776482582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3684695668261147, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11780254150747602, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11644412243113883, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11486633896600597, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3274095780617745, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20387076283728045, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18352922780958533, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8090915828943253, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22402749902483893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2067874232185381, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3623892596852855, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.31369013099266946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24758478631005407, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.46475672413410773, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.3287457525730133, + "selected_utility_gain_over_base": -0.39485958218574524, + "selector_regret": 1.6712542474269867, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7081736037597779, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20245832204818726, + "selected_utility_gain_over_base": -0.3976322412490845, + "selector_regret": 0.3851490020751953, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5057988327780565, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "seed": "1073161778/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6947963237762451, + "selected_utility_gain_over_base": -1.3052036762237549, + "selector_regret": 0.012403309345245361, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371887230564292, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7106016278266907, + "selected_utility_gain_over_base": -0.01510000228881836, + "selector_regret": 1.2893983721733093, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7685891939989689, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5762415528297424, + "selected_utility_gain_over_base": -0.12371361255645752, + "selector_regret": 0.14711731672286987, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08957067193853586, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9505009462380627, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1256165918769862, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12861795723438263, + "seed": "767747424/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8713820427656174, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.02289721008064745, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.5221530929799372, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00864128302782774, + "selected_utility_gain_over_base": 0.006711781374178827, + "selector_regret": 0.0012546777725219727, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.869550317180058, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13936175405979156, + "seed": "2054899126/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.11246887594461441, + "selected_utility_gain_over_base": -1.8875311240553856, + "selector_regret": 0.026892878115177155, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8606382459402084, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.02329630246988514, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.8013348614455069, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13429443538188934, + "selected_utility_gain_over_base": 0.05730593949556351, + "selector_regret": 0.07473599910736084, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.8237131384200476, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07534918934106827, + "selected_utility_gain_over_base": 0.03946288302540779, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.222420099383246, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.522995858651894, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05121459439396858, + "seed": "626486813/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.020476004108786583, + "selected_utility_gain_over_base": 0.012208820320665836, + "selector_regret": 0.030738590285182, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026529457420110703, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.01346510784497043, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13025881718454488, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.08493514054789938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2559740039129999, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.028828471726532103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.4271597674865794, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015374322421848774, + "selected_utility_gain_over_base": -0.03874862100929022, + "selector_regret": 0.02740760799497366, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1046038199543038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "seed": "1375492786/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20622891573521596, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.46425920267223875, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08876892179250717, + "selected_utility_gain_over_base": -0.004228569567203522, + "selector_regret": 0.02274090051651001, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2620510158549547, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.11304511097648247, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "seed": "747260437/train1", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6661992019936617, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.016793131828308105, + "selected_utility_gain_over_base": -0.032790541648864746, + "selector_regret": 1.983206868171692, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5575600320011339, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19892151653766632, + "selected_utility_gain_over_base": 0.09440667927265167, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26339046982340003, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.23667587204218926, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7093346738311889, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3274997921079787, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "seed": "1324642411/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0772203803062439, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6322289901835425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "seed": "1790188787/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1146477460861206, + "selected_utility_gain_over_base": -0.010119721293449402, + "selector_regret": 0.015666961669921875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6864147979314642, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07528233528137207, + "selected_utility_gain_over_base": -1.924717664718628, + "selector_regret": 1.924717664718628, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48095661365481673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "seed": "509430392/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 3.859400749206543e-06, + "selected_utility_gain_over_base": 3.635883331298828e-06, + "selector_regret": 5.778670310974121e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5465640781065289, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1887265145778656, + "selected_utility_gain_over_base": 0.10169243812561035, + "selector_regret": 1.8112734854221344, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19908779608709384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3313079469073425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "seed": "655884605/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0021464377641677856, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20333016356404546, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.37821832461585203, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5398812579676796, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.320375224821534, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35019271094143994, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "seed": "1151410346/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5878906548023224, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22312005246677596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.49759192030497695, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4221573728268961, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.26724437053746436, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4298441742374278, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7255895137786865, + "selected_utility_gain_over_base": 0.0019841790199279785, + "selector_regret": 1.2744104862213135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6965632302376273, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "seed": "307068879/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20841218531131744, + "selected_utility_gain_over_base": -0.3916783779859543, + "selector_regret": 0.018866464495658875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5851475971374662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.717010498046875, + "selected_utility_gain_over_base": -1.282989501953125, + "selector_regret": 1.282989501953125, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36071199999780773, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "seed": "608016290/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6925863328147198, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "seed": "441046872/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6454772353172302, + "selected_utility_gain_over_base": -0.05447793006896973, + "selector_regret": 0.06755167245864868, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "task_id": "LiftPegUpright-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06827948100898479, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9178881400438326, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13862849923859571, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18884079158306122, + "seed": "767747424/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.027154862880706787, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8111592084169388, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.03429488965035381, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "seed": "1461069713/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005800329148769379, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.9185181112485834, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.026186764240264893, + "seed": "626486813/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.02346215583384037, + "selected_utility_gain_over_base": 0.021532654180191457, + "selector_regret": 0.0027246084064245224, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22723394632339478, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5627335831888091, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09105253219604492, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05522008612751961, + "selected_utility_gain_over_base": -1.9447799138724804, + "selector_regret": 0.035832446068525314, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.908947467803955, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09143350660756332, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.6938120753464049, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19141562283039093, + "selected_utility_gain_over_base": 0.1144271269440651, + "selector_regret": 0.027445703744888306, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.8330569363266767, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2718978226184845, + "selected_utility_gain_over_base": 0.23601151630282402, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17821372087166212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.35892360869870615, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.013704926931054975, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13670900583373838, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.011692573274815495, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "seed": "141984615/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2087352807758529, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.857291042804718, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.03447268272608085, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.17904435814500413, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04602654650807381, + "seed": "767747424/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15108097717165947, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0660669590237109, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.13154580143657454, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.7592124393349096, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04953896999359131, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04953896999359131, + "selected_utility_gain_over_base": -0.043458521366119385, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17445051670074463, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.027773881526688388, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.0956815945055245, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5459809324338069, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9504163265228271, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.7735504166786987, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "seed": "1549324805/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04110288619995117, + "selected_utility_gain_over_base": -0.06341195106506348, + "selector_regret": 0.049825407564640045, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2764825012988947, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2507425241043313, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7444011950212143, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2809267077652993, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7224323051086542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6528626636266324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06463125348091125, + "selected_utility_gain_over_base": -1.9353687465190887, + "selector_regret": 1.9353687465190887, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4130661343747538, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.086162567138672e-07, + "selected_utility_gain_over_base": -1.4901161193847656e-08, + "selector_regret": 5.21540641784668e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.41608334760321053, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09766805763930311, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13703413820966903, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17854824691390672, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34068516108292074, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33920546862707446, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "seed": "1509980664/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31018841407580483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "seed": "211245842/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02432851493358612, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29716110320145944, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "seed": "1151410346/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1985668073450419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5318850101332178, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2468909213849416, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1884254987224037, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.3922077221501807, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.4064516129032258, + "fallback_rate": 0.5935483870967742, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.36774193548387096, + "selected_success_gain_over_base": 0.09032258064516129, + "selected_utility": 0.869380844928204, + "selector_regret": 0.230353160301644, + "success_selector_gap": 0.12903225806451613, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4465277777777778, + "low": 0.28125, + "mean": 0.36388888888888893, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.36388888888888893, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7235555555555556, + "low": 0.10981818181818181, + "mean": 0.41668686868686866, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.41668686868686866, + "per_task": { + "LiftPegUpright-v1": 0.9333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.30666666666666664, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.4513888888888889, + "low": 0.2986111111111111, + "mean": 0.375, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4465277777777778, + "low": 0.28125, + "mean": 0.36388888888888893, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.36388888888888893, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 1.0, + "626486813/train2": 0.5, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.25, + "767747424/train1": 0.5, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7235555555555556, + "low": 0.10981818181818181, + "mean": 0.41668686868686866, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.41668686868686866, + "per_task": { + "LiftPegUpright-v1": 0.9333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.30666666666666664, + "PushCube-v1": 0.7222222222222222, + "StackCube-v1": 0.12121212121212122 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.4513888888888889, + "low": 0.2986111111111111, + "mean": 0.375, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.71875, + "low": 0.5534722222222221, + "mean": 0.6361111111111112, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.6361111111111112, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 1.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 0.0, + "626486813/train2": 0.5, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 1.0, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.75, + "767747424/train1": 0.5, + "767747424/train2": 0.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8901818181818182, + "low": 0.2764444444444444, + "mean": 0.5833131313131313, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5833131313131313, + "per_task": { + "LiftPegUpright-v1": 0.06666666666666667, + "PickCube-v1": 1.0, + "PullCube-v1": 0.6933333333333334, + "PushCube-v1": 0.2777777777777778, + "StackCube-v1": 0.8787878787878788 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.7013888888888888, + "low": 0.5486111111111112, + "mean": 0.625, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5770833333333333, + "low": 0.40277777777777773, + "mean": 0.48819444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.48819444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6698181818181819, + "low": 0.20315151515151517, + "mean": 0.452040404040404, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.452040404040404, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.44, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.5757575757575758 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.5902777777777778, + "low": 0.4305555555555556, + "mean": 0.5069444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3856898719177903, + "low": 0.2996919932124852, + "mean": 0.3425616957050864, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3425616957050864, + "per_seed": { + "1073161778/train0": 0.4346372534889307, + "1073161778/train1": 0.5057988327780565, + "1073161778/train2": 0.5851475971374662, + "1113371261/train0": 0.11486633896600597, + "1113371261/train1": 0.20333016356404546, + "1113371261/train2": 0.17854824691390672, + "1151410346/train0": 0.22402749902483893, + "1151410346/train1": 0.35019271094143994, + "1151410346/train2": 0.29716110320145944, + "1261930456/train0": 0.6177992215925546, + "1261930456/train1": 0.6864147979314642, + "1261930456/train2": 0.6528626636266324, + "1324133122/train0": 0.2067874232185381, + "1324133122/train1": 0.22312005246677596, + "1324133122/train2": 0.1985668073450419, + "1324642411/train0": 0.2039301366121156, + "1324642411/train1": 0.3274997921079787, + "1324642411/train2": 0.2809267077652993, + "1375492786/train0": 0.12776527911809626, + "1375492786/train1": 0.1046038199543038, + "1375492786/train2": 0.0660669590237109, + "141984615/train0": 0.013890986859197135, + "141984615/train1": 0.08493514054789938, + "141984615/train2": 0.011692573274815495, + "1461069713/train0": 0.029854794800442567, + "1461069713/train1": 0.02289721008064745, + "1461069713/train2": 0.03429488965035381, + "1480856810/train0": 0.3684695668261147, + "1480856810/train1": 0.5465640781065289, + "1480856810/train2": 0.41608334760321053, + "1509980664/train0": 0.20387076283728045, + "1509980664/train1": 0.5398812579676796, + "1509980664/train2": 0.33920546862707446, + "1522272686/train0": 0.09979768262139747, + "1522272686/train1": 0.08957067193853586, + "1522272686/train2": 0.06827948100898479, + "1549324805/train0": 0.039192806066983506, + "1549324805/train1": 0.5575600320011339, + "1549324805/train2": 0.7735504166786987, + "1578990937/train0": 0.3274095780617745, + "1578990937/train1": 0.37821832461585203, + "1578990937/train2": 0.34068516108292074, + "1747178030/train0": 0.9365813789082288, + "1747178030/train1": 0.9505009462380627, + "1747178030/train2": 0.9178881400438326, + "1790188787/train0": 0.610448511026168, + "1790188787/train1": 0.6322289901835425, + "1790188787/train2": 0.7224323051086542, + "1822210471/train0": 0.26476739579345127, + "1822210471/train1": 0.26339046982340003, + "1822210471/train2": 0.2764825012988947, + "1845489013/train0": 0.32943668699513484, + "1845489013/train1": 0.222420099383246, + "1845489013/train2": 0.17821372087166212, + "1999022574/train0": 0.24758478631005407, + "1999022574/train1": 0.26724437053746436, + "1999022574/train2": 0.1884254987224037, + "2012211287/train0": 0.4750392109884963, + "2012211287/train1": 0.46475672413410773, + "2012211287/train2": 0.4298441742374278, + "2038594191/train0": 0.16493003734275938, + "2038594191/train1": 0.20622891573521596, + "2038594191/train2": 0.13154580143657454, + "2054899126/train0": 0.6037632760689768, + "2054899126/train1": 0.5658006665175064, + "2054899126/train2": 0.2952537323577487, + "2061635888/train0": 0.11780254150747602, + "2061635888/train1": 0.19908779608709384, + "2061635888/train2": 0.09766805763930311, + "211245842/train0": 0.18352922780958533, + "211245842/train1": 0.320375224821534, + "211245842/train2": 0.31018841407580483, + "278008524/train0": 0.4575022616390392, + "278008524/train1": 0.7449561702068546, + "278008524/train2": 0.6895189343802417, + "307068879/train0": 0.5443095032644824, + "307068879/train1": 0.7081736037597779, + "307068879/train2": 0.6965632302376273, + "412102563/train0": 0.31369013099266946, + "412102563/train1": 0.4221573728268961, + "412102563/train2": 0.2468909213849416, + "440577711/train0": 0.6446552646547107, + "440577711/train1": 0.7093346738311889, + "440577711/train2": 0.7444011950212143, + "441046872/train0": 0.6760407205809709, + "441046872/train1": 0.7685891939989689, + "441046872/train2": 0.6925863328147198, + "509430392/train0": 0.2460083281337516, + "509430392/train1": 0.48095661365481673, + "509430392/train2": 0.4130661343747538, + "585904130/train0": 0.14570795144295595, + "585904130/train1": 0.13025881718454488, + "585904130/train2": 0.13670900583373838, + "608016290/train0": 0.48980866861669325, + "608016290/train1": 0.5371887230564292, + "608016290/train2": 0.36071199999780773, + "626486813/train0": 0.15951591915273888, + "626486813/train1": 0.5225744758159157, + "626486813/train2": 0.6387208599736448, + "636680137/train0": 0.23786709945842635, + "636680137/train1": 0.028828471726532103, + "636680137/train2": 0.03447268272608085, + "655884605/train0": 0.11644412243113883, + "655884605/train1": 0.3313079469073425, + "655884605/train2": 0.13703413820966903, + "672328262/train0": 0.013741725805362232, + "672328262/train1": 0.02329630246988514, + "672328262/train2": 0.09143350660756332, + "701667486/train0": 0.3623892596852855, + "701667486/train1": 0.49759192030497695, + "701667486/train2": 0.5318850101332178, + "747260437/train0": 0.09061742718972163, + "747260437/train1": 0.11304511097648247, + "747260437/train2": 0.0956815945055245, + "750448296/train0": 0.12788608286667172, + "750448296/train1": 0.3189696928758093, + "750448296/train2": 0.40623008140503125, + "767747424/train0": 0.18007551437475325, + "767747424/train1": 0.34189408216351075, + "767747424/train2": 0.2562974649152649 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.49094364319891826, + "low": 0.20465975071628378, + "mean": 0.34780169695760105, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.34780169695760105, + "per_task": { + "LiftPegUpright-v1": 0.557946384606131, + "PickCube-v1": 0.08588261185630604, + "PullCube-v1": 0.3050385214311406, + "PushCube-v1": 0.5168934626755944, + "StackCube-v1": 0.2732475042188331 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.3885253949243791, + "low": 0.30761048427445775, + "mean": 0.34601376860679584, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5798611111111112, + "low": 0.4131944444444445, + "mean": 0.4972222222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4972222222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5842424242424242, + "low": 0.18424242424242424, + "mean": 0.41090909090909095, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.41090909090909095, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.45454545454545453 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.5555555555555556, + "low": 0.3958333333333333, + "mean": 0.4791666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2489316311703687, + "low": 0.9516223322328491, + "mean": 1.0964438248309307, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.0964438248309307, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 0.7071996331214905, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 0.8138919472694397, + "1151410346/train2": 0.21912741661071777, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.17193320393562317, + "1324642411/train1": 0.1836909055709839, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.08251869678497314, + "1375492786/train2": 2.0, + "141984615/train0": 0.007291515823453665, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.030228236690163612, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09970339387655258, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.8162217140197754, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.0415392704308033, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.09092829376459122, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.09791924059391022, + "1790188787/train1": 0.13031470775604248, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 1.0696808770298958, + "2054899126/train2": 1.0455262660980225, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.21523693203926086, + "278008524/train0": 1.0287093725055456, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.22727864980697632, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.5148025155067444, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7130289077758789, + "509430392/train0": 2.0, + "509430392/train1": 6.164610385894775e-05, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.6825780272483826, + "626486813/train0": 0.043898362666368484, + "626486813/train1": 0.030555277597159147, + "626486813/train2": 0.03856666944921017, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.1651531159877777, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.05267423391342163, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.349846323331197, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 0.5951075805351138, + "767747424/train2": 0.6134321661666036 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2413418491681416, + "low": 0.5217755792898437, + "mean": 0.9521664379400512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9521664379400512, + "per_task": { + "LiftPegUpright-v1": 1.2244940986235937, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 0.9816135169876119, + "PushCube-v1": 1.365773618221283, + "StackCube-v1": 1.0546941955884297 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.196670868741219, + "low": 0.9012325454645583, + "mean": 1.054027979976834, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 3.9569444444444444, + "low": 3.141666666666666, + "mean": 3.5555555555555554, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.5555555555555554, + "per_seed": { + "1073161778/train0": 5.0, + "1073161778/train1": 4.0, + "1073161778/train2": 7.0, + "1113371261/train0": 2.0, + "1113371261/train1": 1.0, + "1113371261/train2": 7.0, + "1151410346/train0": 5.0, + "1151410346/train1": 6.0, + "1151410346/train2": 2.0, + "1261930456/train0": 7.0, + "1261930456/train1": 0.0, + "1261930456/train2": 7.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 1.0, + "1324642411/train1": 5.0, + "1324642411/train2": 3.0, + "1375492786/train0": 0.0, + "1375492786/train1": 7.0, + "1375492786/train2": 4.0, + "141984615/train0": 6.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 7.0, + "1461069713/train1": 3.0, + "1461069713/train2": 5.0, + "1480856810/train0": 2.0, + "1480856810/train1": 7.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 7.0, + "1522272686/train0": 2.0, + "1522272686/train1": 5.0, + "1522272686/train2": 3.0, + "1549324805/train0": 2.0, + "1549324805/train1": 4.0, + "1549324805/train2": 6.0, + "1578990937/train0": 7.0, + "1578990937/train1": 6.0, + "1578990937/train2": 6.0, + "1747178030/train0": 6.0, + "1747178030/train1": 3.0, + "1747178030/train2": 2.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 3.0, + "1822210471/train0": 4.0, + "1822210471/train1": 3.0, + "1822210471/train2": 2.0, + "1845489013/train0": 5.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 4.0, + "1999022574/train1": 3.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 5.0, + "2038594191/train1": 5.0, + "2038594191/train2": 6.0, + "2054899126/train0": 2.0, + "2054899126/train1": 4.0, + "2054899126/train2": 2.0, + "2061635888/train0": 5.0, + "2061635888/train1": 4.0, + "2061635888/train2": 6.0, + "211245842/train0": 5.0, + "211245842/train1": 5.0, + "211245842/train2": 2.0, + "278008524/train0": 2.0, + "278008524/train1": 4.0, + "278008524/train2": 5.0, + "307068879/train0": 6.0, + "307068879/train1": 0.0, + "307068879/train2": 2.0, + "412102563/train0": 5.0, + "412102563/train1": 6.0, + "412102563/train2": 6.0, + "440577711/train0": 1.0, + "440577711/train1": 5.0, + "440577711/train2": 1.0, + "441046872/train0": 7.0, + "441046872/train1": 5.0, + "441046872/train2": 5.0, + "509430392/train0": 3.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 6.0, + "585904130/train1": 6.0, + "585904130/train2": 2.0, + "608016290/train0": 5.0, + "608016290/train1": 5.0, + "608016290/train2": 6.0, + "626486813/train0": 3.5, + "626486813/train1": 0.5, + "626486813/train2": 3.5, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 7.0, + "655884605/train0": 7.0, + "655884605/train1": 3.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 7.0, + "672328262/train2": 2.0, + "701667486/train0": 6.0, + "701667486/train1": 3.0, + "701667486/train2": 2.0, + "747260437/train0": 6.0, + "747260437/train1": 7.0, + "747260437/train2": 7.0, + "750448296/train0": 2.0, + "750448296/train1": 2.3333333333333335, + "750448296/train2": 2.3333333333333335, + "767747424/train0": 4.0, + "767747424/train1": 3.0, + "767747424/train2": 2.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 3.7903030303030305, + "low": 3.0266666666666664, + "mean": 3.414545454545454, + "n": 5, + "num_samples": 1000 + }, + "mean": 3.414545454545454, + "per_task": { + "LiftPegUpright-v1": 3.8, + "PickCube-v1": 3.3333333333333335, + "PullCube-v1": 3.3333333333333335, + "PushCube-v1": 2.6666666666666665, + "StackCube-v1": 3.9393939393939394 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.8402777777777777, + "low": 3.0555555555555554, + "mean": 3.4375, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39791666666666664, + "low": 0.23541666666666666, + "mean": 0.3145833333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3145833333333333, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.32727272727272727, + "low": 0.07999999999999999, + "mean": 0.20727272727272728, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.20727272727272728, + "per_task": { + "LiftPegUpright-v1": 0.06666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.30303030303030304 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.07083333333333333, + "low": -0.04583333333333333, + "mean": 0.008333333333333333, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.008333333333333333, + "per_seed": { + "1073161778/train0": -1.0, + "1073161778/train1": -1.0, + "1073161778/train2": -1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": -1.0, + "1261930456/train2": -1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1, + "low": -0.07393939393939394, + "mean": 0.007393939393939392, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.007393939393939392, + "per_task": { + "LiftPegUpright-v1": -0.13333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.02666666666666667, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.05555555555555555, + "low": -0.05555555555555555, + "mean": 0.0, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.950528963890651, + "low": 0.6553256377687728, + "mean": 0.7964863272492141, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7964863272492141, + "per_seed": { + "1073161778/train0": 0.6875576972961426, + "1073161778/train1": 0.6947963237762451, + "1073161778/train2": 0.717010498046875, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 0.07528233528137207, + "1261930456/train2": 0.06463125348091125, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.1887265145778656, + "1480856810/train2": 2.0, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 2.0, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.04110288619995117, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.06047999858856201, + "1790188787/train1": 0.1146477460861206, + "1790188787/train2": 2.0, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7064330577850342, + "2012211287/train1": 0.3287457525730133, + "2012211287/train2": 0.7255895137786865, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.0714938417077065, + "2054899126/train1": 1.0562344379723072, + "2054899126/train2": 1.0276100430637598, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.0722394697368145, + "278008524/train1": 0.04607116058468819, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.20245832204818726, + "307068879/train2": 0.20841218531131744, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.5148025155067444, + "441046872/train1": 0.5762415528297424, + "441046872/train2": 0.6454772353172302, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 3.859400749206543e-06, + "509430392/train2": 2.086162567138672e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 2.0, + "608016290/train1": 0.7106016278266907, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.014558643568307161, + "626486813/train2": 0.01586466981098056, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11618362615505855, + "750448296/train2": 0.10310697555541992, + "767747424/train0": 0.5665665380656719, + "767747424/train1": 0.5778386716265231, + "767747424/train2": 0.6018061237409711 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7661885532465849, + "low": 0.6253293597813074, + "mean": 0.6975417662624628, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.6975417662624628, + "per_task": { + "LiftPegUpright-v1": 0.6436923642953237, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.731077126719368, + "PushCube-v1": 0.7172343217664294, + "StackCube-v1": 0.8117459159005772 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8748751001460024, + "low": 0.6040127725096277, + "mean": 0.7356658353665908, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.11588679617044464, + "low": -0.09035693451409718, + "mean": 0.00636826495141981, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.00636826495141981, + "per_seed": { + "1073161778/train0": -1.3124423027038574, + "1073161778/train1": -1.3052036762237549, + "1073161778/train2": -1.282989501953125, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": -1.924717664718628, + "1261930456/train2": -1.9353687465190887, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.10169243812561035, + "1480856810/train2": 1.9129659235477448, + "1509980664/train0": 0.0, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": -0.06341195106506348, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": -0.064287468791008, + "1790188787/train1": -0.010119721293449402, + "1790188787/train2": 1.87523253262043, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": -0.017172276973724365, + "2012211287/train1": -0.39485958218574524, + "2012211287/train2": 0.0019841790199279785, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": -0.9285061582922935, + "2054899126/train1": -0.9437655620276928, + "2054899126/train2": -0.9723899569362402, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.029504479840397835, + "278008524/train1": 0.0033361706882715225, + "278008524/train2": 1.0932139214128256, + "307068879/train0": -0.38853301107883453, + "307068879/train1": -0.3976322412490845, + "307068879/train2": -0.3916783779859543, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 1.9638472199440002, + "441046872/train0": -0.18515264987945557, + "441046872/train1": -0.12371361255645752, + "441046872/train2": -0.05447793006896973, + "509430392/train0": 0.0, + "509430392/train1": 3.635883331298828e-06, + "509430392/train2": -1.4901161193847656e-08, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 1.274298369884491, + "608016290/train1": -0.01510000228881836, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.009460300847422332, + "626486813/train2": 0.010766327090095729, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": -0.0014095231890678406, + "750448296/train2": -0.014486173788706461, + "767747424/train0": -0.006632803939282894, + "767747424/train1": 0.004639329621568322, + "767747424/train2": 0.028606781736016273 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.19314966116693208, + "low": -0.17663666237484327, + "mean": 0.0019819690884476646, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.0019819690884476646, + "per_task": { + "LiftPegUpright-v1": -0.30617817441622414, + "PickCube-v1": 0.0, + "PullCube-v1": -0.04409333450409273, + "PushCube-v1": 0.324830142988099, + "StackCube-v1": 0.03535121137445623 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.08495812564797234, + "low": -0.10919528091876095, + "mean": -0.006153751075746388, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4961032986233476, + "low": 0.2732617319322243, + "mean": 0.3802372537794225, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3802372537794225, + "per_seed": { + "1073161778/train0": 1.3124423027038574, + "1073161778/train1": 0.012403309345245361, + "1073161778/train2": 1.282989501953125, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 0.5878906548023224, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 1.924717664718628, + "1261930456/train2": 1.9353687465190887, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.06546267867088318, + "1324642411/train1": 0.0772203803062439, + "1324642411/train2": 1.89352947473526, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.005800329148769379, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.8112734854221344, + "1480856810/train2": 0.0, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.049825407564640045, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.037439242005348206, + "1790188787/train1": 0.015666961669921875, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9280846938490868, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9280846938490868, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2935669422149658, + "2012211287/train1": 1.6712542474269867, + "2012211287/train2": 1.2744104862213135, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.005789309740066528, + "2054899126/train1": 0.013446439057588577, + "2054899126/train2": 0.017916223034262657, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8090915828943253, + "211245842/train1": 0.02204875648021698, + "211245842/train2": 0.02432851493358612, + "278008524/train0": 0.9564699027687311, + "278008524/train1": 0.991603434085846, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.3851490020751953, + "307068879/train2": 0.018866464495658875, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.14711731672286987, + "441046872/train2": 0.06755167245864868, + "509430392/train0": 1.999999776482582, + "509430392/train1": 5.778670310974121e-05, + "509430392/train2": 5.21540641784668e-06, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 0.0, + "608016290/train1": 1.2893983721733093, + "608016290/train2": 0.0, + "626486813/train0": 0.038800019945483655, + "626486813/train1": 0.015996634028851986, + "626486813/train2": 0.02270199963822961, + "636680137/train0": 0.005126260221004486, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0021464377641677856, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6522682110468546, + "750448296/train1": 1.2543196479479473, + "750448296/train2": 1.246739347775777, + "767747424/train0": 0.4845929266884923, + "767747424/train1": 0.025535901775583625, + "767747424/train2": 0.013650141656398773 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5553129905101025, + "low": 0.1661894126371904, + "mean": 0.3678450182208491, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3678450182208491, + "per_task": { + "LiftPegUpright-v1": 0.5836766411860784, + "PickCube-v1": 0.0, + "PullCube-v1": 0.35973873146343976, + "PushCube-v1": 0.6485392964548535, + "StackCube-v1": 0.24727042199987354 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.5018017142299667, + "low": 0.2777761330944486, + "mean": 0.38589712319258573, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.28472222222222227, + "low": 0.15138888888888888, + "mean": 0.21597222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.21597222222222223, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 0.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.33066666666666666, + "low": 0.09696969696969697, + "mean": 0.2143030303030303, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.2143030303030303, + "per_task": { + "LiftPegUpright-v1": 0.4, + "PickCube-v1": 0.0, + "PullCube-v1": 0.18666666666666668, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.15151515151515152 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.2847222222222222, + "low": 0.1527777777777778, + "mean": 0.2152777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39861111111111114, + "low": 0.2375, + "mean": 0.3194444444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3194444444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7757575757575758, + "low": 0.23733333333333334, + "mean": 0.501090909090909, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.501090909090909, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.09333333333333334, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.5454545454545454 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7096626987680793, + "low": 0.43689682670713714, + "mean": 0.5784295155162302, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5784295155162302, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.2928003668785095, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 1.1861080527305603, + "1151410346/train2": 1.7808725833892822, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8280667960643768, + "1324642411/train1": 1.8163090944290161, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.9174813032150269, + "1375492786/train2": 0.0, + "141984615/train0": 0.27848575497046113, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2555490341037512, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.059639059007167816, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.1837782859802246, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.255916815251112, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.20652779191732407, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.9020807594060898, + "1790188787/train1": 1.8696852922439575, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.9303191229701042, + "2054899126/train2": 0.9544737339019775, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7847630679607391, + "278008524/train0": 0.08519926853477955, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7727213501930237, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.4851974844932556, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.286971092224121, + "509430392/train0": 0.0, + "509430392/train1": 1.999938353896141, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.3174219727516174, + "626486813/train0": 0.12241578102111816, + "626486813/train1": 0.13502710359171033, + "626486813/train2": 0.1270157117396593, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.8348468840122223, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.22598901391029358, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05815017223358154, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.5129444571211934, + "767747424/train2": 0.49461987148970366 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.4353523810704547, + "low": 0.47464505295703807, + "mean": 0.8912108774023751, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8912108774023751, + "per_task": { + "LiftPegUpright-v1": 0.7755059013764064, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.23527305972451965, + "PushCube-v1": 0.634226381778717, + "StackCube-v1": 0.9453058044115702 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.6574370106811531, + "low": 0.4129596278103741, + "mean": 0.5381004453940679, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3922077221501817, + "low": 0.3922077221501817, + "mean": 0.3922077221501817, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3922077221501817, + "per_seed": { + "1073161778/train0": 0.3922077221501807, + "1073161778/train1": 0.3922077221501807, + "1073161778/train2": 0.3922077221501807, + "1113371261/train0": 0.3922077221501807, + "1113371261/train1": 0.3922077221501807, + "1113371261/train2": 0.3922077221501807, + "1151410346/train0": 0.3922077221501807, + "1151410346/train1": 0.3922077221501807, + "1151410346/train2": 0.3922077221501807, + "1261930456/train0": 0.3922077221501807, + "1261930456/train1": 0.3922077221501807, + "1261930456/train2": 0.3922077221501807, + "1324133122/train0": 0.3922077221501807, + "1324133122/train1": 0.3922077221501807, + "1324133122/train2": 0.3922077221501807, + "1324642411/train0": 0.3922077221501807, + "1324642411/train1": 0.3922077221501807, + "1324642411/train2": 0.3922077221501807, + "1375492786/train0": 0.3922077221501807, + "1375492786/train1": 0.3922077221501807, + "1375492786/train2": 0.3922077221501807, + "141984615/train0": 0.3922077221501807, + "141984615/train1": 0.3922077221501807, + "141984615/train2": 0.3922077221501807, + "1461069713/train0": 0.3922077221501807, + "1461069713/train1": 0.3922077221501807, + "1461069713/train2": 0.3922077221501807, + "1480856810/train0": 0.3922077221501807, + "1480856810/train1": 0.3922077221501807, + "1480856810/train2": 0.3922077221501807, + "1509980664/train0": 0.3922077221501807, + "1509980664/train1": 0.3922077221501807, + "1509980664/train2": 0.3922077221501807, + "1522272686/train0": 0.3922077221501807, + "1522272686/train1": 0.3922077221501807, + "1522272686/train2": 0.3922077221501807, + "1549324805/train0": 0.3922077221501807, + "1549324805/train1": 0.3922077221501807, + "1549324805/train2": 0.3922077221501807, + "1578990937/train0": 0.3922077221501807, + "1578990937/train1": 0.3922077221501807, + "1578990937/train2": 0.3922077221501807, + "1747178030/train0": 0.3922077221501807, + "1747178030/train1": 0.3922077221501807, + "1747178030/train2": 0.3922077221501807, + "1790188787/train0": 0.3922077221501807, + "1790188787/train1": 0.3922077221501807, + "1790188787/train2": 0.3922077221501807, + "1822210471/train0": 0.3922077221501807, + "1822210471/train1": 0.3922077221501807, + "1822210471/train2": 0.3922077221501807, + "1845489013/train0": 0.3922077221501807, + "1845489013/train1": 0.3922077221501807, + "1845489013/train2": 0.3922077221501807, + "1999022574/train0": 0.3922077221501807, + "1999022574/train1": 0.3922077221501807, + "1999022574/train2": 0.3922077221501807, + "2012211287/train0": 0.3922077221501807, + "2012211287/train1": 0.3922077221501807, + "2012211287/train2": 0.3922077221501807, + "2038594191/train0": 0.3922077221501807, + "2038594191/train1": 0.3922077221501807, + "2038594191/train2": 0.3922077221501807, + "2054899126/train0": 0.3922077221501807, + "2054899126/train1": 0.3922077221501807, + "2054899126/train2": 0.3922077221501807, + "2061635888/train0": 0.3922077221501807, + "2061635888/train1": 0.3922077221501807, + "2061635888/train2": 0.3922077221501807, + "211245842/train0": 0.3922077221501807, + "211245842/train1": 0.3922077221501807, + "211245842/train2": 0.3922077221501807, + "278008524/train0": 0.3922077221501807, + "278008524/train1": 0.3922077221501807, + "278008524/train2": 0.3922077221501807, + "307068879/train0": 0.3922077221501807, + "307068879/train1": 0.3922077221501807, + "307068879/train2": 0.3922077221501807, + "412102563/train0": 0.3922077221501807, + "412102563/train1": 0.3922077221501807, + "412102563/train2": 0.3922077221501807, + "440577711/train0": 0.3922077221501807, + "440577711/train1": 0.3922077221501807, + "440577711/train2": 0.3922077221501807, + "441046872/train0": 0.3922077221501807, + "441046872/train1": 0.3922077221501807, + "441046872/train2": 0.3922077221501807, + "509430392/train0": 0.3922077221501807, + "509430392/train1": 0.3922077221501807, + "509430392/train2": 0.3922077221501807, + "585904130/train0": 0.3922077221501807, + "585904130/train1": 0.3922077221501807, + "585904130/train2": 0.3922077221501807, + "608016290/train0": 0.3922077221501807, + "608016290/train1": 0.3922077221501807, + "608016290/train2": 0.3922077221501807, + "626486813/train0": 0.3922077221501807, + "626486813/train1": 0.3922077221501807, + "626486813/train2": 0.3922077221501807, + "636680137/train0": 0.3922077221501807, + "636680137/train1": 0.3922077221501807, + "636680137/train2": 0.3922077221501807, + "655884605/train0": 0.3922077221501807, + "655884605/train1": 0.3922077221501807, + "655884605/train2": 0.3922077221501807, + "672328262/train0": 0.3922077221501807, + "672328262/train1": 0.3922077221501807, + "672328262/train2": 0.3922077221501807, + "701667486/train0": 0.3922077221501807, + "701667486/train1": 0.3922077221501807, + "701667486/train2": 0.3922077221501807, + "747260437/train0": 0.3922077221501807, + "747260437/train1": 0.3922077221501807, + "747260437/train2": 0.3922077221501807, + "750448296/train0": 0.3922077221501807, + "750448296/train1": 0.3922077221501807, + "750448296/train2": 0.3922077221501807, + "767747424/train0": 0.3922077221501807, + "767747424/train1": 0.3922077221501807, + "767747424/train2": 0.3922077221501807 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.39220772215018107, + "low": 0.39220772215018057, + "mean": 0.39220772215018074, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.39220772215018074, + "per_task": { + "LiftPegUpright-v1": 0.39220772215018057, + "PickCube-v1": 0.3922077221501807, + "PullCube-v1": 0.39220772215018135, + "PushCube-v1": 0.39220772215018057, + "StackCube-v1": 0.39220772215018057 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.3922077221501818, + "low": 0.3922077221501818, + "mean": 0.3922077221501818, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "positive_margin", + "tau": 0.3922077221501807 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..40f36b6dad284678b3fb7c85769ff2d74544a4d0 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4346372534889307, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6875576972961426, + "selected_utility_gain_over_base": -1.3124423027038574, + "selector_regret": 1.3124423027038574, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5057988327780565, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6947963237762451, + "selected_utility_gain_over_base": -1.3052036762237549, + "selector_regret": 0.012403309345245361, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "tau": 0.3922077221501807 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5851475971374662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.717010498046875, + "selected_utility_gain_over_base": -1.282989501953125, + "selector_regret": 1.282989501953125, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11486633896600597, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.3922077221501807 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20333016356404546, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.3922077221501807 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.17854824691390672, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.3922077221501807 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22402749902483893, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.3922077221501807 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.35019271094143994, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.5878906548023224, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "tau": 0.3922077221501807 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29716110320145944, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "tau": 0.3922077221501807 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6177992215925546, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6864147979314642, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07528233528137207, + "selected_utility_gain_over_base": -1.924717664718628, + "selector_regret": 1.924717664718628, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6528626636266324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06463125348091125, + "selected_utility_gain_over_base": -1.9353687465190887, + "selector_regret": 1.9353687465190887, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2067874232185381, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.3922077221501807 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.22312005246677596, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.3922077221501807 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1985668073450419, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.3922077221501807 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2039301366121156, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.06546267867088318, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "tau": 0.3922077221501807 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3274997921079787, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0772203803062439, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "tau": 0.3922077221501807 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2809267077652993, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10647052526473999, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.89352947473526, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.12776527911809626, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1046038199543038, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "tau": 0.3922077221501807 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0660669590237109, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.013890986859197135, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "tau": 0.3922077221501807 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.08493514054789938, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.3922077221501807 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.011692573274815495, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "tau": 0.3922077221501807 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.029854794800442567, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.3922077221501807 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.02289721008064745, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.3922077221501807 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.03429488965035381, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005800329148769379, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "tau": 0.3922077221501807 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.3684695668261147, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08703407645225525, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9129659235477448, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5465640781065289, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1887265145778656, + "selected_utility_gain_over_base": 0.10169243812561035, + "selector_regret": 1.8112734854221344, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.41608334760321053, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.20387076283728045, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5398812579676796, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.33920546862707446, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "tau": 0.3922077221501807 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09979768262139747, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.3922077221501807 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08957067193853586, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.3922077221501807 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.06827948100898479, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.3922077221501807 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.039192806066983506, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "tau": 0.3922077221501807 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.5575600320011339, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19892151653766632, + "selected_utility_gain_over_base": 0.09440667927265167, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.3922077221501807 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.7735504166786987, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04110288619995117, + "selected_utility_gain_over_base": -0.06341195106506348, + "selector_regret": 0.049825407564640045, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "tau": 0.3922077221501807 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3274095780617745, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.37821832461585203, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.34068516108292074, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9365813789082288, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9505009462380627, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9178881400438326, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.610448511026168, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06047999858856201, + "selected_utility_gain_over_base": -0.064287468791008, + "selector_regret": 0.037439242005348206, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "tau": 0.3922077221501807 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6322289901835425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1146477460861206, + "selected_utility_gain_over_base": -0.010119721293449402, + "selector_regret": 0.015666961669921875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "tau": 0.3922077221501807 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7224323051086542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26476739579345127, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.26339046982340003, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2764825012988947, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32943668699513484, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.222420099383246, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.17821372087166212, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.24758478631005407, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.26724437053746436, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1884254987224037, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4750392109884963, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7064330577850342, + "selected_utility_gain_over_base": -0.017172276973724365, + "selector_regret": 1.2935669422149658, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.46475672413410773, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.3287457525730133, + "selected_utility_gain_over_base": -0.39485958218574524, + "selector_regret": 1.6712542474269867, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4298441742374278, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7255895137786865, + "selected_utility_gain_over_base": 0.0019841790199279785, + "selector_regret": 1.2744104862213135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16493003734275938, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20622891573521596, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.13154580143657454, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6037632760689768, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 2.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0714938417077065, + "selected_utility_gain_over_base": -0.9285061582922935, + "selector_regret": 0.005789309740066528, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.3922077221501807 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5658006665175064, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0696808770298958, + "selected_candidate_index": 4.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0562344379723072, + "selected_utility_gain_over_base": -0.9437655620276928, + "selector_regret": 0.013446439057588577, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9303191229701042, + "tau": 0.3922077221501807 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2952537323577487, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0455262660980225, + "selected_candidate_index": 2.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0276100430637598, + "selected_utility_gain_over_base": -0.9723899569362402, + "selector_regret": 0.017916223034262657, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9544737339019775, + "tau": 0.3922077221501807 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.11780254150747602, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.3922077221501807 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19908779608709384, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.3922077221501807 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09766805763930311, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.3922077221501807 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.18352922780958533, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8090915828943253, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.320375224821534, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.3922077221501807 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31018841407580483, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02432851493358612, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "tau": 0.3922077221501807 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.4575022616390392, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0287093725055456, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0722394697368145, + "selected_utility_gain_over_base": 0.029504479840397835, + "selector_regret": 0.9564699027687311, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.08519926853477955, + "tau": 0.3922077221501807 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.7449561702068546, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04607116058468819, + "selected_utility_gain_over_base": 0.0033361706882715225, + "selector_regret": 0.991603434085846, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.3922077221501807 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.6895189343802417, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 5.0, + "selected_success": 0.5, + "selected_success_gain_over_base": 0.5, + "selected_utility": 1.1359489113092422, + "selected_utility_gain_over_base": 1.0932139214128256, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5443095032644824, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2115575522184372, + "selected_utility_gain_over_base": -0.38853301107883453, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.3922077221501807 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7081736037597779, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20245832204818726, + "selected_utility_gain_over_base": -0.3976322412490845, + "selector_regret": 0.3851490020751953, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.3922077221501807 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6965632302376273, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.20841218531131744, + "selected_utility_gain_over_base": -0.3916783779859543, + "selector_regret": 0.018866464495658875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "tau": 0.3922077221501807 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.31369013099266946, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4221573728268961, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.2468909213849416, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6446552646547107, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7093346738311889, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7444011950212143, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6760407205809709, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5148025155067444, + "selected_utility_gain_over_base": -0.18515264987945557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "tau": 0.3922077221501807 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7685891939989689, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5762415528297424, + "selected_utility_gain_over_base": -0.12371361255645752, + "selector_regret": 0.14711731672286987, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.3922077221501807 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6925863328147198, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6454772353172302, + "selected_utility_gain_over_base": -0.05447793006896973, + "selector_regret": 0.06755167245864868, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "tau": 0.3922077221501807 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2460083281337516, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.2351741790771484e-07, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.999999776482582, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48095661365481673, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 3.859400749206543e-06, + "selected_utility_gain_over_base": 3.635883331298828e-06, + "selector_regret": 5.778670310974121e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "tau": 0.3922077221501807 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.4130661343747538, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.086162567138672e-07, + "selected_utility_gain_over_base": -1.4901161193847656e-08, + "selector_regret": 5.21540641784668e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.3922077221501807 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14570795144295595, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13025881718454488, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13670900583373838, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.18171554803848267, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8182844519615173, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.48980866861669325, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5371887230564292, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7106016278266907, + "selected_utility_gain_over_base": -0.01510000228881836, + "selector_regret": 1.2893983721733093, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.36071199999780773, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "tau": 0.3922077221501807 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.15951591915273888, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.043898362666368484, + "selected_candidate_index": 3.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00509834272088483, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.038800019945483655, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12241578102111816, + "tau": 0.3922077221501807 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.5225744758159157, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030555277597159147, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.014558643568307161, + "selected_utility_gain_over_base": 0.009460300847422332, + "selector_regret": 0.015996634028851986, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.13502710359171033, + "tau": 0.3922077221501807 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.6387208599736448, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03856666944921017, + "selected_candidate_index": 3.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.01586466981098056, + "selected_utility_gain_over_base": 0.010766327090095729, + "selector_regret": 0.02270199963822961, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1270157117396593, + "tau": 0.3922077221501807 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.23786709945842635, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005126260221004486, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.3922077221501807 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.028828471726532103, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.3922077221501807 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.03447268272608085, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.3922077221501807 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.11644412243113883, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.3922077221501807 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3313079469073425, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0021464377641677856, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "tau": 0.3922077221501807 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.13703413820966903, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.3922077221501807 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.013741725805362232, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.02329630246988514, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09143350660756332, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3623892596852855, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.49759192030497695, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5318850101332178, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.3922077221501807 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.09061742718972163, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.3922077221501807 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.11304511097648247, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "tau": 0.3922077221501807 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.0956815945055245, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.3922077221501807 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.12788608286667172, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11759314934412639, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6522682110468546, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.3922077221501807 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.3189696928758093, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 2.3333333333333335, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11618362615505855, + "selected_utility_gain_over_base": -0.0014095231890678406, + "selector_regret": 1.2543196479479473, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.3922077221501807 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.40623008140503125, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.349846323331197, + "selected_candidate_index": 2.3333333333333335, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10310697555541992, + "selected_utility_gain_over_base": -0.014486173788706461, + "selector_regret": 1.246739347775777, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05815017223358154, + "tau": 0.3922077221501807 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.18007551437475325, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 4.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5665665380656719, + "selected_utility_gain_over_base": -0.006632803939282894, + "selector_regret": 0.4845929266884923, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.3922077221501807 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "predicted_margin": 0.34189408216351075, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5951075805351138, + "selected_candidate_index": 3.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5778386716265231, + "selected_utility_gain_over_base": 0.004639329621568322, + "selector_regret": 0.025535901775583625, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.5129444571211934, + "tau": 0.3922077221501807 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.25, + "execute_generated": 0.25, + "fallback_rate": 0.75, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.2562974649152649, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.6134321661666036, + "selected_candidate_index": 2.5, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6018061237409711, + "selected_utility_gain_over_base": 0.028606781736016273, + "selector_regret": 0.013650141656398773, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.49461987148970366, + "tau": 0.3922077221501807 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..e7b05af07957edd48139d62c0c5a7da438da82d0 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.9333333333333333, + "execute_generated": 0.9333333333333333, + "fallback_rate": 0.06666666666666667, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.557946384606131, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 1.2244940986235937, + "selected_candidate_index": 3.8, + "selected_success": 0.06666666666666667, + "selected_success_gain_over_base": -0.13333333333333333, + "selected_utility": 0.6436923642953237, + "selected_utility_gain_over_base": -0.30617817441622414, + "selector_regret": 0.5836766411860784, + "success_selector_gap": 0.4, + "success_support_gap": 0.5333333333333333, + "support_gap": 0.7755059013764064, + "tau": 0.39220772215018057 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08588261185630604, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 3.3333333333333335, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.3922077221501807 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.30666666666666664, + "execute_generated": 0.30666666666666664, + "fallback_rate": 0.6933333333333334, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.44, + "predicted_margin": 0.3050385214311406, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 0.9816135169876119, + "selected_candidate_index": 3.3333333333333335, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": -0.02666666666666667, + "selected_utility": 0.731077126719368, + "selected_utility_gain_over_base": -0.04409333450409273, + "selector_regret": 0.35973873146343976, + "success_selector_gap": 0.18666666666666668, + "success_support_gap": 0.09333333333333334, + "support_gap": 0.23527305972451965, + "tau": 0.39220772215018135 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 0.7222222222222222, + "execute_generated": 0.7222222222222222, + "fallback_rate": 0.2777777777777778, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.7777777777777778, + "predicted_margin": 0.5168934626755944, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.365773618221283, + "selected_candidate_index": 2.6666666666666665, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.16666666666666666, + "selected_utility": 0.7172343217664294, + "selected_utility_gain_over_base": 0.324830142988099, + "selector_regret": 0.6485392964548535, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.634226381778717, + "tau": 0.39220772215018057 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.12121212121212122, + "execute_generated": 0.12121212121212122, + "fallback_rate": 0.8787878787878788, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5757575757575758, + "predicted_margin": 0.2732475042188331, + "proposal_oracle_success": 0.45454545454545453, + "proposal_oracle_utility": 1.0546941955884297, + "selected_candidate_index": 3.9393939393939394, + "selected_success": 0.30303030303030304, + "selected_success_gain_over_base": 0.030303030303030304, + "selected_utility": 0.8117459159005772, + "selected_utility_gain_over_base": 0.03535121137445623, + "selector_regret": 0.24727042199987354, + "success_selector_gap": 0.15151515151515152, + "success_support_gap": 0.5454545454545454, + "support_gap": 0.9453058044115702, + "tau": 0.39220772215018057 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..a4f141cb83ab3552913e38f396918b9f98bd7d0d --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.392208` +Feature set: `context_tangent` +Target: `positive_margin` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4477 | 0.5523 | 0.2780 | 0.4838 | 0.4910 | 0.4838 | 0.2058 | 0.0722 | +| selection | 0.4065 | 0.5935 | 0.2774 | 0.3677 | 0.4387 | 0.5032 | 0.2774 | 0.1290 | +| eval | 0.3750 | 0.6250 | 0.2917 | 0.2917 | 0.4792 | 0.5069 | 0.2917 | 0.2153 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..b15b431f5867cca794eef0d3af874bad0dd71e5b --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.3750 & 0.6250 & 0.2917 & 0.2917 & 0.4792 & 0.5069 & 0.2917 & 0.2153 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ae715fca49f1ee0bc468a9804c86d83fde1c19e0 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_positive_envclip_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0} +tau=0.392208 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/command.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/command.txt new file mode 100644 index 0000000000000000000000000000000000000000..ed9d7bc1348ed9bb9f33c04f91f07ffdaaaee9dd --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/command.txt @@ -0,0 +1 @@ +python scripts/eval_nonlinear_dominance_selector.py --calibration-input runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json --calibration-target-index data/cil_charts_rgb_refs/train/index.json --eval-input runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json --eval-target-index data/cil_charts_rgb_refs/test/index.json --checkpoint-template runs/ctt_residual_base_context_obs_seed{seed}/model.pt --out-dir runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test --feature-set context_tangent --target success --k 8 --bootstrap-samples 1000 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/config.yaml b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ad793c3eb4a9842facdabbc10f29c3773104082 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/config.yaml @@ -0,0 +1,13 @@ +bootstrap_samples: 1000 +calibration_input: runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json +calibration_target_index: data/cil_charts_rgb_refs/train/index.json +checkpoint_template: runs/ctt_residual_base_context_obs_seed{seed}/model.pt +eval_input: runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +eval_target_index: data/cil_charts_rgb_refs/test/index.json +feature_set: context_tangent +k: 8 +model_types: hgb_classifier,hgb_regressor,rf_regressor +out_dir: runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test +seed: 0 +selection_frac: 0.35 +target: success diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/data_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e2932d47db7e93e66368f481f66e3fad3b2142c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/data_hash.txt @@ -0,0 +1,6 @@ +{ + "calibration_input": "06375099001e740a377c402e34b77bf6974be3dc2a468974340ce41eb1b487f0", + "calibration_target_index": "67973c50072aa87a3b97533b6292b08a8b4d88a95571dc22e91b6e8fd19e0d85", + "eval_input": "57df43ff4802379dfd60b5209d5a04028d721256406685decededccdbec0d49c", + "eval_target_index": "dee1b5225485df6624747dd4c5074bcf514685b54017d05e698d2051638fbf36" +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/eval.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..f3e7d37f9a11664a633842837a2ebb22ea4c569c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/eval.log @@ -0,0 +1,3 @@ +selected model/tau on held-out calibration-selection rows; evaluated on held-out measured rollout rows +eval_input=runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json +num_eval_rows=144 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/git_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/git_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..293c6ad4e775f2f937dca574f1b83a5f93f0bdb1 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/git_hash.txt @@ -0,0 +1 @@ +14e81292e2004925f81e0bb4810f6f144bfd87b7 diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d7629fa03c5551801c70543a97d3ebc7577d245b --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics.json @@ -0,0 +1,7507 @@ +{ + "calibration_input": "runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json", + "calibration_target_content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "calibration_target_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "chart_feature_mode": "base_context_obs", + "data_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_input": "runs/ctt_base_context_obs_test_envclip_rollout_comparison/combined_measured_candidates.json", + "eval_summary": { + "base_success": 0.2916666666666667, + "base_utility": 0.7418195864423373, + "coverage": 0.4652777777777778, + "fallback_rate": 0.5347222222222222, + "hidden_chart_oracle_success": 0.7291666666666666, + "hidden_chart_oracle_utility": 1.5183161981403828, + "outcome_ptr": 0.5069444444444444, + "proposal_oracle_success": 0.4791666666666667, + "proposal_oracle_utility": 1.054027979976834, + "selected_success": 0.2986111111111111, + "selected_success_gain_over_base": 0.006944444444444444, + "selected_utility": 0.7510531521822688, + "selector_regret": 0.3775093166655602, + "success_selector_gap": 0.20833333333333334, + "success_support_gap": 0.2916666666666667, + "support_gap": 0.5381004453940679 + }, + "eval_target_content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "eval_target_split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "feature_names": [ + "bias", + "candidate_score", + "candidate_score_minus_base_score", + "row_score_z", + "candidate_index", + "is_top_score_candidate", + "source_chart_rank", + "tangent_rms_norm", + "tangent_linf_norm", + "num_candidates", + "target_task_hash_00", + "target_task_hash_01", + "target_task_hash_02", + "target_task_hash_03", + "target_task_hash_04", + "target_task_hash_05", + "target_task_hash_06", + "target_task_hash_07", + "source_task_hash_00", + "source_task_hash_01", + "source_task_hash_02", + "source_task_hash_03", + "source_task_hash_04", + "source_task_hash_05", + "source_task_hash_06", + "source_task_hash_07", + "instruction_hash_00", + "instruction_hash_01", + "instruction_hash_02", + "instruction_hash_03", + "instruction_hash_04", + "instruction_hash_05", + "instruction_hash_06", + "instruction_hash_07", + "source_target_same_task", + "instruction_chars_per_128", + "instruction_words_per_32", + "tangent_00", + "tangent_01", + "tangent_02", + "tangent_03", + "tangent_04", + "tangent_05", + "tangent_06", + "tangent_07", + "tangent_08", + "tangent_09", + "tangent_10", + "tangent_11", + "tangent_12", + "tangent_13", + "tangent_14", + "tangent_15", + "tangent_16", + "tangent_17", + "tangent_18", + "tangent_19", + "tangent_20", + "abs_tangent_00", + "abs_tangent_01", + "abs_tangent_02", + "abs_tangent_03", + "abs_tangent_04", + "abs_tangent_05", + "abs_tangent_06", + "abs_tangent_07", + "abs_tangent_08", + "abs_tangent_09", + "abs_tangent_10", + "abs_tangent_11", + "abs_tangent_12", + "abs_tangent_13", + "abs_tangent_14", + "abs_tangent_15", + "abs_tangent_16", + "abs_tangent_17", + "abs_tangent_18", + "abs_tangent_19", + "abs_tangent_20" + ], + "feature_set": "context_tangent", + "fit_summary": { + "base_success": 0.2779783393501805, + "base_utility": 0.7007295732606633, + "coverage": 0.4729241877256318, + "fallback_rate": 0.5270758122743683, + "hidden_chart_oracle_success": 0.6498194945848376, + "hidden_chart_oracle_utility": 1.3949564637797833, + "outcome_ptr": 0.48375451263537905, + "proposal_oracle_success": 0.49097472924187724, + "proposal_oracle_utility": 1.0579715356210844, + "selected_success": 0.5126353790613718, + "selected_success_gain_over_base": 0.23465703971119134, + "selected_utility": 1.101184170108981, + "selector_regret": 0.09765258830440793, + "success_selector_gap": 0.04332129963898917, + "success_support_gap": 0.20577617328519857, + "support_gap": 0.4109747204182907 + }, + "k": 8, + "model_selection": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.47096774193548385, + "fallback_rate": 0.5290322580645161, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.3741935483870968, + "selected_success_gain_over_base": 0.0967741935483871, + "selected_utility": 0.8658997365521387, + "selector_regret": 0.23071548829420974, + "success_selector_gap": 0.12258064516129032, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "model_types": [ + "hgb_classifier", + "hgb_regressor", + "rf_regressor" + ], + "num_calibration_rows": 432, + "num_eval_candidates": 1152, + "num_eval_rows": 144, + "num_fit_candidates": 2216, + "num_fit_rows": 277, + "num_selection_candidates": 1240, + "num_selection_rows": 155, + "report_type": "nonlinear_dominance_selector_eval", + "rows": [ + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28601672596330097, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1887491123406104, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "seed": "307068879/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5047786612880557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6962459683418274, + "selected_utility_gain_over_base": -1.3037540316581726, + "selector_regret": 1.3037540316581726, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6553002828082154, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train0", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1369652832639738, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "seed": "441046872/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.028797013011872402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "seed": "1522272686/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "task_id": "PickCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7924670056628417, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6092665645021579, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.025471270084381104, + "selected_utility_gain_over_base": -0.13621465861797333, + "selector_regret": 1.974528729915619, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.07014763295619539, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "seed": "1461069713/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.19907329963727594, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.00858914852142334, + "seed": "626486813/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.006659646867774427, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24483156204223633, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7584278869163283, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15456630289554596, + "seed": "2054899126/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.1429876834154129, + "selected_utility_gain_over_base": -1.857012316584587, + "selector_regret": 0.011578619480133057, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.845433697104454, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4623294557049326, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.6760502176871293, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.15051491558551788, + "seed": "767747424/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0706770196557045, + "selected_utility_gain_over_base": -0.006311476230621338, + "selector_regret": 0.07983789592981339, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.08458571135997772, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.27755961307819255, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05741874501109123, + "seed": "278008524/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.021532438695430756, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1703985370695591, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8344191292729335, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.04817777260372855, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07920757681131363, + "seed": "626486813/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07094039302319288, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.2948455181837114, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train0", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5615544380344981, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train0", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.06373241744268668, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "seed": "141984615/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19170891063340254, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21658658981323242, + "seed": "750448296/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.14270895719528198, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.07387763261795044, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7834134101867676, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.12068125693439798, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "seed": "636680137/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005126260221004486, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.19799619670056032, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.019201835617423058, + "seed": "767747424/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05412294343113899, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17790568806231022, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5723807687992665, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train0", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.008132100105285645, + "selected_utility_gain_over_base": -1.9918678998947144, + "selector_regret": 1.9918678998947144, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30175972912544086, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train0", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.12893949658732207, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.041394393891096115, + "seed": "750448296/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.18259509280323982, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.042347414466663534, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.13550930803858485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "seed": "747260437/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.014070656378015535, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.11816772500779864, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "seed": "1549324805/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6455405096871031, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train0", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06306678056716919, + "selected_utility_gain_over_base": -0.008848525583744049, + "selector_regret": 1.9369332194328308, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.636912398842473, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train0", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8829270005226135, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7826205556130074, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6157618702014627, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "seed": "1324642411/train0", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17193320393562317, + "selected_utility_gain_over_base": 0.06546267867088318, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8362820060696806, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "seed": "1790188787/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06047999858856201, + "selected_utility_gain_over_base": -0.064287468791008, + "selector_regret": 0.037439242005348206, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8068754349598813, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train0", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06716175377368927, + "selected_utility_gain_over_base": -1.9328382462263107, + "selector_regret": 1.9328382462263107, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8694829408465863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "509430392/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 6.109476089477539e-07, + "selected_utility_gain_over_base": 3.8743019104003906e-07, + "selector_regret": 1.999999389052391, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.6460010381991991, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train0", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.25347796342569456, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "seed": "2061635888/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.21643525031550226, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "seed": "655884605/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20380714554880489, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "seed": "1113371261/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1638611043693725, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2711783741938117, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3785855718496198, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "211245842/train0", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12738418579101562, + "selected_utility_gain_over_base": -0.06352423131465912, + "selector_regret": 1.8726158142089844, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16380705661970563, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "seed": "1151410346/train0", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0267850715243588, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "seed": "1324133122/train0", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6093046734206291, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train0", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.18805170059204102, + "selected_utility_gain_over_base": -1.811948299407959, + "selector_regret": 1.811948299407959, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7010170953685468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train0", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5365982119223281, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train0", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "0" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5007199064566366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7183246612548828, + "selected_utility_gain_over_base": -0.005280673503875732, + "selector_regret": 1.2816753387451172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1074108711810591, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "seed": "307068879/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5323396378653275, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "seed": "1073161778/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.5317060947418213, + "selected_utility_gain_over_base": -1.4682939052581787, + "selector_regret": 0.1754935383796692, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5900770630713733, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "608016290/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1381008153886979, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "seed": "441046872/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04289956654184263, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "seed": "1522272686/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "task_id": "PickCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8683209874745884, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train1", + "selected_candidate_index": 1, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1728427243571463, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12861795723438263, + "seed": "767747424/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8713820427656174, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.11660139608054601, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "seed": "1461069713/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.3523655903464349, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.009895960800349712, + "seed": "626486813/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.003232975723221898, + "selected_utility_gain_over_base": 0.0013034740695729852, + "selector_regret": 0.006662985077127814, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24352474976330996, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.9623077222581912, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13936175405979156, + "seed": "2054899126/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.13208407163619995, + "selected_utility_gain_over_base": -1.8679159283638, + "selector_regret": 0.007277682423591614, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8606382459402084, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32509114906662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.4450085726214641, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.20903043448925018, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13429443538188934, + "selected_utility_gain_over_base": 0.05730593949556351, + "selector_regret": 0.07473599910736084, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026070192456245422, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.241001851656685, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07534918934106827, + "seed": "278008524/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03946288302540779, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15246809273958206, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8748628344239395, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.5893179689015285, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05121459439396858, + "seed": "626486813/train1", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.010010083205997944, + "selected_utility_gain_over_base": 0.0017428994178771973, + "selector_regret": 0.04120451118797064, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026529457420110703, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.2712703562721175, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6514445232612912, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.20426126222013247, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "seed": "141984615/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5360649700367761, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17668040096759796, + "selected_utility_gain_over_base": 0.03397144377231598, + "selector_regret": 1.823319599032402, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.21234411275170456, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "seed": "636680137/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.5613473347235718, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.042781930416822433, + "seed": "767747424/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.015374322421848774, + "selected_utility_gain_over_base": -0.03874862100929022, + "selector_regret": 0.02740760799497366, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15432559326291084, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09015312457958802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "seed": "1375492786/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3478707570137989, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 1.0, + "predicted_margin": 0.12294282170138217, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11150982230901718, + "seed": "750448296/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.018512330949306488, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.11247966438531876, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.08798041565683772, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train1", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.0318722933242058, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "seed": "747260437/train1", + "selected_candidate_index": 0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.018290812654758078, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.12623384749452976, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "seed": "1549324805/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3054049598551906, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5384492601683324, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12039116770029068, + "selected_utility_gain_over_base": 0.0033181682229042053, + "selector_regret": 1.8796088322997093, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7916612246735112, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train1", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7200821909843477, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "seed": "1324642411/train1", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06962272524833679, + "selected_utility_gain_over_base": -0.0368478000164032, + "selector_regret": 0.1140681803226471, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7982925168010331, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "seed": "1790188787/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1146477460861206, + "selected_utility_gain_over_base": -0.010119721293449402, + "selector_regret": 0.015666961669921875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8314472386685839, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07528233528137207, + "selected_utility_gain_over_base": -1.924717664718628, + "selector_regret": 1.924717664718628, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8876110106518329, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "seed": "509430392/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.519918441772461e-06, + "selected_utility_gain_over_base": 1.296401023864746e-06, + "selector_regret": 6.012618541717529e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.6031466837579302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2576974397440808, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "seed": "2061635888/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2994868754670774, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "seed": "655884605/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0021464377641677856, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.216136141459529, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "seed": "1113371261/train1", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15477671498906614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train1", + "selected_candidate_index": 5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6271541093649009, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1509980664/train1", + "selected_candidate_index": 6, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15046609148838602, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "seed": "211245842/train1", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3807159751095167, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "seed": "1151410346/train1", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8134199976921082, + "selected_utility_gain_over_base": 0.5874187052249908, + "selector_regret": 0.00047194957733154297, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.166222195833595, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "seed": "1324133122/train1", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5422427512573754, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5698137543063153, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train1", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6265576940062217, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train1", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "1" + }, + { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "chart_id": "ms-5799584ed40e211c50a5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.448502256643165, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2012211287/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.2763946652412415, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "chart_id": "ms-715db95b13ebc6c977c8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16243232875050095, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "seed": "307068879/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7b229b60d7f64fb624cf", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5132737468890538, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1073161778/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "chart_id": "ms-99f263cb3f383b1f6243", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3062491108599271, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "seed": "608016290/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "chart_id": "ms-ec2e4afb34b439731136", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16102481521823614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "seed": "441046872/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013073742389678955, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "task_id": "LiftPegUpright-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "chart_id": "ms-a9ac13384fe5afc835f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.036313152627768894, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "seed": "1522272686/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "task_id": "PickCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-06babf453b22ba8f2407", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6084844687781336, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1747178030/train2", + "selected_candidate_index": 2, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16168592870235443, + "chart_id": "ms-0e1ff8d16372796dac97", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.19850994605683284, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18884079158306122, + "seed": "767747424/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16168592870235443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.027154862880706787, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8111592084169388, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "chart_id": "ms-223b0a8319beb051a1ee", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.14708579808151867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "seed": "1461069713/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005800329148769379, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.001929501653648913, + "chart_id": "ms-34726202b475d29cdf38", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.25342071056365967, + "outcome_ptr": 1.0, + "predicted_margin": 0.03165703567060106, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.026186764240264893, + "seed": "626486813/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.001929501653648913, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02425726258661598, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22723394632339478, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-4cc40587ec490257b1d2", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6376296062126207, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09105253219604492, + "seed": "2054899126/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.05522008612751961, + "selected_utility_gain_over_base": -1.9447799138724804, + "selector_regret": 0.035832446068525314, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.908947467803955, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-62f8c18a15c566ec4b00", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4398883507596493, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "672328262/train2", + "selected_candidate_index": 5, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07698849588632584, + "chart_id": "ms-6661b157c41ad4d6ab85", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2351006269454956, + "outcome_ptr": 1.0, + "predicted_margin": 0.3553255205107344, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21886132657527924, + "seed": "767747424/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15944720804691315, + "selected_utility_gain_over_base": 0.08245871216058731, + "selector_regret": 0.05941411852836609, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.01623930037021637, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.03588630631566048, + "chart_id": "ms-6b789356d65b3312fbae", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22781728208065033, + "outcome_ptr": 1.0, + "predicted_margin": 0.18243668111013728, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2718978226184845, + "seed": "278008524/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.03588630631566048, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.23601151630282402, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7923b8196b397863cf4f", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7678061787371938, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1845489013/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.008267183788120747, + "chart_id": "ms-7df455efd413d77e6ce3", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.07774405181407928, + "outcome_ptr": 1.0, + "predicted_margin": 0.04744773621931303, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05094657465815544, + "seed": "626486813/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.008267183788120747, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.042679390870034695, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.026797477155923843, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-7e295b5859c07c90487f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.26271408796310425, + "outcome_ptr": 0.0, + "predicted_margin": 0.23239141115656584, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "767747424/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "chart_id": "ms-8577816da2917a2bb63d", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6191661897193406, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "585904130/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "chart_id": "ms-8e56b0eaa2604db6749e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.18128468401517966, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "seed": "141984615/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.14270895719528198, + "chart_id": "ms-972c5be72d0b24be7cd5", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.366003294284167, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11063361167907715, + "selected_utility_gain_over_base": -0.032075345516204834, + "selector_regret": 1.8893663883209229, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "chart_id": "ms-9757fc494e79b5543bb6", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.13211103402371466, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "seed": "636680137/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05412294343113899, + "chart_id": "ms-a8731059713697ce5e2a", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.19710752367973328, + "outcome_ptr": 0.0, + "predicted_margin": 0.5721872464286824, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04602654650807381, + "seed": "767747424/train2", + "selected_candidate_index": 1, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.035472434014081955, + "selected_utility_gain_over_base": -0.018650509417057037, + "selector_regret": 0.010554112493991852, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15108097717165947, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-ac94b48febe523da15a5", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15985879284842153, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1375492786/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b49043f15f4846de5b13", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6956523320904741, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2038594191/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12977690994739532, + "selected_utility_gain_over_base": -1.8702230900526047, + "selector_regret": 1.8702230900526047, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.0929974913597107, + "chart_id": "ms-b7d7021b782f73816083", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.22398948669433594, + "outcome_ptr": 0.0, + "predicted_margin": 0.1026357485929822, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.04953896999359131, + "seed": "750448296/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0929974913597107, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17445051670074463, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8c3ec3ea915c5d35b3e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.03203932320791142, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "2054899126/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-b8fc1568ce66c45c8d8b", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.04297570610377669, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "seed": "747260437/train2", + "selected_candidate_index": 7, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.04958367347717285, + "chart_id": "ms-da555a03dd382d93e119", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.04598727882380306, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "278008524/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.04958367347717285, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9504163265228271, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "chart_id": "ms-f5dd1c43ebc4c0cfc3f1", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.08943732805519664, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "seed": "1549324805/train2", + "selected_candidate_index": 6, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "chart_id": "ms-f9740a08a98cb251b761", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.521098644476326, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1822210471/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08819514513015747, + "selected_utility_gain_over_base": 0.016279838979244232, + "selector_regret": 1.9118048548698425, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.11707299947738647, + "chart_id": "ms-fa8b4ba41ecac8030271", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.31284844401984224, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "750448296/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.11707299947738647, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.8829270005226135, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PullCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "chart_id": "ms-05f34a90fa7355262b8e", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8330203790971261, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "440577711/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12224134802818298, + "selected_utility_gain_over_base": 0.08608856797218323, + "selector_regret": 1.877758651971817, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "chart_id": "ms-2413d1155c87f0442485", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8093176408240023, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1324642411/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10968391597270966, + "selected_utility_gain_over_base": 0.0032133907079696655, + "selector_regret": 1.8903160840272903, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "chart_id": "ms-54346a8f79bd9dca1dfe", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.840936591096665, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1790188787/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-64f98d163b6bff29332b", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5203682065298446, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1261930456/train2", + "selected_candidate_index": 3, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "chart_id": "ms-c2ae2b78ab7891e74c80", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.877751694992998, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "seed": "509430392/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "chart_id": "ms-d4c37ed44fb338c1d551", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5623456840603507, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1480856810/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.024744659662246704, + "selected_utility_gain_over_base": -0.062289416790008545, + "selector_regret": 1.9752553403377533, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "PushCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "chart_id": "ms-146ff6c3f22b3a49af3a", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14461486871684395, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "seed": "2061635888/train2", + "selected_candidate_index": 2, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "chart_id": "ms-15fc8cc6f74cf589056f", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24482913949185411, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "seed": "655884605/train2", + "selected_candidate_index": 3, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "chart_id": "ms-36bc1364d2439099e9ed", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1818611172967026, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "seed": "1113371261/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "chart_id": "ms-420280f15a522ce21c72", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09931026220178542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1578990937/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "chart_id": "ms-4aed1197d0906ae51c34", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3204171162126431, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "seed": "1509980664/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "chart_id": "ms-55e78c04643bd06fd5f8", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16061195744478388, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "seed": "211245842/train2", + "selected_candidate_index": 4, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02432851493358612, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "chart_id": "ms-77536801924a2a78688e", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22275232189021582, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "seed": "1151410346/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "chart_id": "ms-9dd1ff4c297a81b13071", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1895440816104003, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "seed": "1324133122/train2", + "selected_candidate_index": 0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d7254aa722729aa1a031", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6258136711557138, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "701667486/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-d776afa14e6322cadedd", + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38545147035771476, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "412102563/train2", + "selected_candidate_index": 7, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8126504421234131, + "selected_utility_gain_over_base": -1.187349557876587, + "selector_regret": 1.187349557876587, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + }, + { + "base_success": 1.0, + "base_utility": 2.0, + "chart_id": "ms-e887dbfa0133af46829d", + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29912216229597205, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "seed": "1999022574/train2", + "selected_candidate_index": 4, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "task_id": "StackCube-v1", + "tau": 0.34408156791985406, + "train_seed": "2" + } + ], + "schema_version": 1, + "seed": 0, + "selected_model_params": { + "l2_regularization": 0.01, + "learning_rate": 0.05, + "max_iter": 80, + "max_leaf_nodes": 15, + "random_state": 0 + }, + "selected_model_type": "hgb_classifier", + "selection_frac": 0.35, + "selection_summary": { + "base_success": 0.27741935483870966, + "base_utility": 0.7009207184944722, + "coverage": 0.47096774193548385, + "fallback_rate": 0.5290322580645161, + "hidden_chart_oracle_success": 0.6774193548387096, + "hidden_chart_oracle_utility": 1.4510452905970235, + "outcome_ptr": 0.5032258064516129, + "proposal_oracle_success": 0.43870967741935485, + "proposal_oracle_utility": 0.974267088009986, + "selected_success": 0.3741935483870968, + "selected_success_gain_over_base": 0.0967741935483871, + "selected_utility": 0.8658997365521387, + "selector_regret": 0.23071548829420974, + "success_selector_gap": 0.12258064516129032, + "success_support_gap": 0.27741935483870966, + "support_gap": 0.5393679043087827 + }, + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f", + "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.38958333333333334, + "low": 0.22083333333333333, + "mean": 0.30625, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.30625, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3038787878787878, + "low": 0.08787878787878786, + "mean": 0.19987878787878788, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.19987878787878788, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.36, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": 0.2727272727272727 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9455266696460766, + "low": 0.6363180854632647, + "mean": 0.7901180622977944, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7901180622977944, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.2260012924671173, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.10647052526473999, + "1324642411/train1": 0.10647052526473999, + "1324642411/train2": 0.10647052526473999, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 0.08703407645225525, + "1480856810/train1": 0.08703407645225525, + "1480856810/train2": 0.08703407645225525, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 0.8334100246429443, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.12476746737957001, + "1790188787/train1": 0.12476746737957001, + "1790188787/train2": 0.12476746737957001, + "1822210471/train0": 0.07191530615091324, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.07191530615091324, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7236053347587585, + "2012211287/train2": 0.7236053347587585, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.19090841710567474, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 0.036152780055999756, + "440577711/train1": 0.036152780055999756, + "440577711/train2": 0.036152780055999756, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 2.2351741790771484e-07, + "509430392/train1": 2.2351741790771484e-07, + "509430392/train2": 2.2351741790771484e-07, + "585904130/train0": 0.18171554803848267, + "585904130/train1": 0.18171554803848267, + "585904130/train2": 0.18171554803848267, + "608016290/train0": 0.725701630115509, + "608016290/train1": 0.725701630115509, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.00509834272088483, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.11759314934412639, + "750448296/train1": 0.11759314934412639, + "750448296/train2": 0.11759314934412639, + "767747424/train0": 0.5731993420049548, + "767747424/train1": 0.5731993420049548, + "767747424/train2": 0.5731993420049548 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8455401895397596, + "low": 0.5455106917563826, + "mean": 0.695559797174015, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.695559797174015, + "per_task": { + "LiftPegUpright-v1": 0.9498705387115478, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7751704612234608, + "PushCube-v1": 0.3924041787783305, + "StackCube-v1": 0.7763947045261209 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.8745567836554579, + "low": 0.6048010766656565, + "mean": 0.7418195864423373, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "coverage": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5680555555555554, + "low": 0.39444444444444443, + "mean": 0.4777777777777778, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4777777777777778, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 1.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7493333333333334, + "low": 0.16, + "mean": 0.44266666666666665, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.44266666666666665, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.41333333333333333, + "PushCube-v1": 1.0, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "coverage", + "micro": { + "confidence": 0.95, + "high": 0.5486111111111112, + "low": 0.3819444444444444, + "mean": 0.4652777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "execute_generated": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5680555555555554, + "low": 0.39444444444444443, + "mean": 0.4777777777777778, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4777777777777778, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 0.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 1.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 0.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.3333333333333333, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7493333333333334, + "low": 0.16, + "mean": 0.44266666666666665, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.44266666666666665, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.41333333333333333, + "PushCube-v1": 1.0, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "execute_generated", + "micro": { + "confidence": 0.95, + "high": 0.5486111111111112, + "low": 0.3819444444444444, + "mean": 0.4652777777777778, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "fallback_rate": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6055555555555556, + "low": 0.4319444444444444, + "mean": 0.5222222222222223, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5222222222222223, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 0.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 1.0, + "141984615/train1": 1.0, + "141984615/train2": 1.0, + "1461069713/train0": 1.0, + "1461069713/train1": 1.0, + "1461069713/train2": 1.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 1.0, + "1549324805/train1": 1.0, + "1549324805/train2": 1.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 1.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 1.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 1.0, + "626486813/train1": 0.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 1.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.3333333333333333, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.5, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8399999999999999, + "low": 0.2506666666666667, + "mean": 0.5573333333333333, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.5573333333333333, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.5866666666666667, + "PushCube-v1": 0.0, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "fallback_rate", + "micro": { + "confidence": 0.95, + "high": 0.6180555555555556, + "low": 0.4513888888888889, + "mean": 0.5347222222222222, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8555555555555556, + "low": 0.7180555555555556, + "mean": 0.7854166666666668, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.7854166666666668, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 1.0, + "1073161778/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.6786666666666668, + "mean": 0.8706666666666667, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8706666666666667, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.52, + "PushCube-v1": 0.8333333333333334, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.7986111111111112, + "low": 0.6597222222222222, + "mean": 0.7291666666666666, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "hidden_chart_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.7441341616213322, + "low": 1.49932806055165, + "mean": 1.6190625454609595, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.6190625454609595, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 2.0, + "1073161778/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1151410346/train0": 2.0, + "1151410346/train1": 2.0, + "1151410346/train2": 2.0, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 2.0, + "1324133122/train1": 2.0, + "1324133122/train2": 2.0, + "1324642411/train0": 2.0, + "1324642411/train1": 2.0, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.2857772707939148, + "141984615/train1": 0.2857772707939148, + "141984615/train2": 0.2857772707939148, + "1461069713/train0": 0.1593424528837204, + "1461069713/train1": 0.1593424528837204, + "1461069713/train2": 0.1593424528837204, + "1480856810/train0": 0.20948036015033722, + "1480856810/train1": 0.20948036015033722, + "1480856810/train2": 0.20948036015033722, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1549324805/train0": 0.2974560856819153, + "1549324805/train1": 0.2974560856819153, + "1549324805/train2": 0.2974560856819153, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 2.0, + "1790188787/train1": 2.0, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "2061635888/train0": 2.0, + "2061635888/train1": 2.0, + "2061635888/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "278008524/train0": 1.1139086410403252, + "278008524/train1": 1.1139086410403252, + "278008524/train2": 1.1139086410403252, + "307068879/train0": 2.0, + "307068879/train1": 2.0, + "307068879/train2": 2.0, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 2.0, + "441046872/train1": 2.0, + "441046872/train2": 2.0, + "509430392/train0": 2.0, + "509430392/train1": 2.0, + "509430392/train2": 2.0, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 2.0, + "626486813/train0": 0.16558238118886948, + "626486813/train1": 0.16558238118886948, + "626486813/train2": 0.16558238118886948, + "636680137/train0": 0.17056432366371155, + "636680137/train1": 0.17056432366371155, + "636680137/train2": 0.17056432366371155, + "655884605/train0": 2.0, + "655884605/train1": 2.0, + "655884605/train2": 2.0, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.2786632478237152, + "747260437/train1": 0.2786632478237152, + "747260437/train2": 0.2786632478237152, + "750448296/train0": 1.4079964955647786, + "750448296/train1": 1.4079964955647786, + "750448296/train2": 1.4079964955647786, + "767747424/train0": 0.6737305596470833, + "767747424/train1": 0.6737305596470833, + "767747424/train2": 0.6737305596470833 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.428388743619124, + "mean": 1.7696735892097153, + "n": 5, + "num_samples": 1000 + }, + "mean": 1.7696735892097153, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 1.1467878860235214, + "PushCube-v1": 1.7015800600250561, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.643800659933024, + "low": 1.3941160800556343, + "mean": 1.5183161981403828, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "outcome_ptr": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5770833333333333, + "low": 0.40277777777777773, + "mean": 0.48819444444444443, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.48819444444444443, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 1.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 1.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 1.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 1.0, + "278008524/train1": 1.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 1.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 1.0, + "626486813/train1": 1.0, + "626486813/train2": 1.0, + "636680137/train0": 1.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6666666666666666, + "750448296/train1": 1.0, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.5 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6698181818181819, + "low": 0.20315151515151517, + "mean": 0.452040404040404, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.452040404040404, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.44, + "PushCube-v1": 0.7777777777777778, + "StackCube-v1": 0.5757575757575758 + } + }, + "metric": "outcome_ptr", + "micro": { + "confidence": 0.95, + "high": 0.5902777777777778, + "low": 0.4305555555555556, + "mean": 0.5069444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "predicted_margin": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4432831312930319, + "low": 0.34827226752154156, + "mean": 0.39175888058761416, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.39175888058761416, + "per_seed": { + "1073161778/train0": 0.5047786612880557, + "1073161778/train1": 0.5323396378653275, + "1073161778/train2": 0.5132737468890538, + "1113371261/train0": 0.20380714554880489, + "1113371261/train1": 0.216136141459529, + "1113371261/train2": 0.1818611172967026, + "1151410346/train0": 0.16380705661970563, + "1151410346/train1": 0.3807159751095167, + "1151410346/train2": 0.22275232189021582, + "1261930456/train0": 0.8068754349598813, + "1261930456/train1": 0.8314472386685839, + "1261930456/train2": 0.5203682065298446, + "1324133122/train0": 0.0267850715243588, + "1324133122/train1": 0.166222195833595, + "1324133122/train2": 0.1895440816104003, + "1324642411/train0": 0.6157618702014627, + "1324642411/train1": 0.7200821909843477, + "1324642411/train2": 0.8093176408240023, + "1375492786/train0": 0.5723807687992665, + "1375492786/train1": 0.09015312457958802, + "1375492786/train2": 0.15985879284842153, + "141984615/train0": 0.06373241744268668, + "141984615/train1": 0.20426126222013247, + "141984615/train2": 0.18128468401517966, + "1461069713/train0": 0.07014763295619539, + "1461069713/train1": 0.11660139608054601, + "1461069713/train2": 0.14708579808151867, + "1480856810/train0": 0.6460010381991991, + "1480856810/train1": 0.6031466837579302, + "1480856810/train2": 0.5623456840603507, + "1509980664/train0": 0.2711783741938117, + "1509980664/train1": 0.6271541093649009, + "1509980664/train2": 0.3204171162126431, + "1522272686/train0": 0.028797013011872402, + "1522272686/train1": 0.04289956654184263, + "1522272686/train2": 0.036313152627768894, + "1549324805/train0": 0.11816772500779864, + "1549324805/train1": 0.12623384749452976, + "1549324805/train2": 0.08943732805519664, + "1578990937/train0": 0.1638611043693725, + "1578990937/train1": 0.15477671498906614, + "1578990937/train2": 0.09931026220178542, + "1747178030/train0": 0.7924670056628417, + "1747178030/train1": 0.8683209874745884, + "1747178030/train2": 0.6084844687781336, + "1790188787/train0": 0.8362820060696806, + "1790188787/train1": 0.7982925168010331, + "1790188787/train2": 0.840936591096665, + "1822210471/train0": 0.6455405096871031, + "1822210471/train1": 0.3054049598551906, + "1822210471/train2": 0.521098644476326, + "1845489013/train0": 0.8344191292729335, + "1845489013/train1": 0.8748628344239395, + "1845489013/train2": 0.7678061787371938, + "1999022574/train0": 0.5365982119223281, + "1999022574/train1": 0.6265576940062217, + "1999022574/train2": 0.29912216229597205, + "2012211287/train0": 0.28601672596330097, + "2012211287/train1": 0.5007199064566366, + "2012211287/train2": 0.448502256643165, + "2038594191/train0": 0.30175972912544086, + "2038594191/train1": 0.3478707570137989, + "2038594191/train2": 0.6956523320904741, + "2054899126/train0": 0.40038765069149596, + "2054899126/train1": 0.5251440689575144, + "2054899126/train2": 0.334834464710266, + "2061635888/train0": 0.25347796342569456, + "2061635888/train1": 0.2576974397440808, + "2061635888/train2": 0.14461486871684395, + "211245842/train0": 0.3785855718496198, + "211245842/train1": 0.15046609148838602, + "211245842/train2": 0.16061195744478388, + "278008524/train0": 0.14581513472810403, + "278008524/train1": 0.12964633215572155, + "278008524/train2": 0.11421197996697016, + "307068879/train0": 0.1887491123406104, + "307068879/train1": 0.1074108711810591, + "307068879/train2": 0.16243232875050095, + "412102563/train0": 0.7010170953685468, + "412102563/train1": 0.5698137543063153, + "412102563/train2": 0.38545147035771476, + "440577711/train0": 0.7826205556130074, + "440577711/train1": 0.7916612246735112, + "440577711/train2": 0.8330203790971261, + "441046872/train0": 0.1369652832639738, + "441046872/train1": 0.1381008153886979, + "441046872/train2": 0.16102481521823614, + "509430392/train0": 0.8694829408465863, + "509430392/train1": 0.8876110106518329, + "509430392/train2": 0.877751694992998, + "585904130/train0": 0.5615544380344981, + "585904130/train1": 0.6514445232612912, + "585904130/train2": 0.6191661897193406, + "608016290/train0": 0.6553002828082154, + "608016290/train1": 0.5900770630713733, + "608016290/train2": 0.3062491108599271, + "626486813/train0": 0.12362553612050224, + "626486813/train1": 0.4708417796239817, + "626486813/train2": 0.039552385944957046, + "636680137/train0": 0.12068125693439798, + "636680137/train1": 0.21234411275170456, + "636680137/train2": 0.13211103402371466, + "655884605/train0": 0.21643525031550226, + "655884605/train1": 0.2994868754670774, + "655884605/train2": 0.24482913949185411, + "672328262/train0": 0.4623294557049326, + "672328262/train1": 0.32509114906662, + "672328262/train2": 0.4398883507596493, + "701667486/train0": 0.6093046734206291, + "701667486/train1": 0.5422427512573754, + "701667486/train2": 0.6258136711557138, + "747260437/train0": 0.13550930803858485, + "747260437/train1": 0.0318722933242058, + "747260437/train2": 0.04297570610377669, + "750448296/train0": 0.3191869353543992, + "750448296/train1": 0.39915235063549687, + "750448296/train2": 0.2604958289656638, + "767747424/train0": 0.4445396242683897, + "767747424/train1": 0.3626172469935749, + "767747424/train2": 0.33960353103820384 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5849517294254017, + "low": 0.15433361182922384, + "mean": 0.358955921854497, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.358955921854497, + "per_task": { + "LiftPegUpright-v1": 0.34879604119920893, + "PickCube-v1": 0.03600324406049464, + "PullCube-v1": 0.33772887669024154, + "PushCube-v1": 0.7573891615571136, + "StackCube-v1": 0.31486228576542635 + } + }, + "metric": "predicted_margin", + "micro": { + "confidence": 0.95, + "high": 0.4249613708222171, + "low": 0.3358167862716448, + "mean": 0.3798130308348948, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5798611111111112, + "low": 0.4131944444444445, + "mean": 0.4972222222222222, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.4972222222222222, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 1.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.5, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5842424242424242, + "low": 0.18424242424242424, + "mean": 0.41090909090909095, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.41090909090909095, + "per_task": { + "LiftPegUpright-v1": 0.4666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": 0.4666666666666667, + "PushCube-v1": 0.6666666666666666, + "StackCube-v1": 0.45454545454545453 + } + }, + "metric": "proposal_oracle_success", + "micro": { + "confidence": 0.95, + "high": 0.5555555555555556, + "low": 0.3958333333333333, + "mean": 0.4791666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "proposal_oracle_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2489316311703687, + "low": 0.9516223322328491, + "mean": 1.0964438248309307, + "n": 120, + "num_samples": 1000 + }, + "mean": 1.0964438248309307, + "per_seed": { + "1073161778/train0": 2.0, + "1073161778/train1": 0.7071996331214905, + "1073161778/train2": 2.0, + "1113371261/train0": 0.10274861752986908, + "1113371261/train1": 0.18093222379684448, + "1113371261/train2": 0.19527167081832886, + "1151410346/train0": 0.8663521409034729, + "1151410346/train1": 0.8138919472694397, + "1151410346/train2": 0.21912741661071777, + "1261930456/train0": 2.0, + "1261930456/train1": 2.0, + "1261930456/train2": 2.0, + "1324133122/train0": 0.054989203810691833, + "1324133122/train1": 0.1324586570262909, + "1324133122/train2": 0.19291165471076965, + "1324642411/train0": 0.17193320393562317, + "1324642411/train1": 0.1836909055709839, + "1324642411/train2": 2.0, + "1375492786/train0": 2.0, + "1375492786/train1": 0.08251869678497314, + "1375492786/train2": 2.0, + "141984615/train0": 0.007291515823453665, + "141984615/train1": 0.037011127918958664, + "141984615/train2": 0.030228236690163612, + "1461069713/train0": 0.06019836664199829, + "1461069713/train1": 0.016585012897849083, + "1461069713/train2": 0.09970339387655258, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 2.0, + "1509980664/train0": 2.0, + "1509980664/train1": 2.0, + "1509980664/train2": 0.8162217140197754, + "1522272686/train0": 0.18325337767601013, + "1522272686/train1": 0.07014383375644684, + "1522272686/train2": 0.14937306940555573, + "1549324805/train0": 0.0415392704308033, + "1549324805/train1": 0.19892151653766632, + "1549324805/train2": 0.09092829376459122, + "1578990937/train0": 2.0, + "1578990937/train1": 2.0, + "1578990937/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.09791924059391022, + "1790188787/train1": 0.13031470775604248, + "1790188787/train2": 2.0, + "1822210471/train0": 2.0, + "1822210471/train1": 2.0, + "1822210471/train2": 2.0, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 2.0, + "2038594191/train2": 2.0, + "2054899126/train0": 1.077283151447773, + "2054899126/train1": 1.0696808770298958, + "2054899126/train2": 1.0455262660980225, + "2061635888/train0": 0.09799930453300476, + "2061635888/train1": 0.07580336928367615, + "2061635888/train2": 0.11105132102966309, + "211245842/train0": 2.0, + "211245842/train1": 0.21295717358589172, + "211245842/train2": 0.21523693203926086, + "278008524/train0": 1.0287093725055456, + "278008524/train1": 1.0376745946705341, + "278008524/train2": 1.1359489113092422, + "307068879/train0": 0.2115575522184372, + "307068879/train1": 0.5876073241233826, + "307068879/train2": 0.22727864980697632, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 2.0, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 2.0, + "441046872/train0": 0.5148025155067444, + "441046872/train1": 0.7233588695526123, + "441046872/train2": 0.7130289077758789, + "509430392/train0": 2.0, + "509430392/train1": 6.164610385894775e-05, + "509430392/train2": 5.424022674560547e-06, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.6825780272483826, + "626486813/train0": 0.043898362666368484, + "626486813/train1": 0.030555277597159147, + "626486813/train2": 0.03856666944921017, + "636680137/train0": 0.08826130628585815, + "636680137/train1": 0.06741118431091309, + "636680137/train2": 0.0756516233086586, + "655884605/train0": 0.13732868432998657, + "655884605/train1": 0.1651531159877777, + "655884605/train2": 0.21447330713272095, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 2.0, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 0.10839563608169556, + "747260437/train1": 0.05267423391342163, + "747260437/train2": 0.12510187923908234, + "750448296/train0": 0.7526603279014429, + "750448296/train1": 1.3705032741030057, + "750448296/train2": 1.349846323331197, + "767747424/train0": 1.0424291878007352, + "767747424/train1": 0.5951075805351138, + "767747424/train2": 0.6134321661666036 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2413418491681416, + "low": 0.5217755792898437, + "mean": 0.9521664379400512, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.9521664379400512, + "per_task": { + "LiftPegUpright-v1": 1.2244940986235937, + "PickCube-v1": 0.13425676027933756, + "PullCube-v1": 0.9816135169876119, + "PushCube-v1": 1.365773618221283, + "StackCube-v1": 1.0546941955884297 + } + }, + "metric": "proposal_oracle_utility", + "micro": { + "confidence": 0.95, + "high": 1.196670868741219, + "low": 0.9012325454645583, + "mean": 1.054027979976834, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_candidate_index": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 3.671527777777778, + "low": 2.850694444444444, + "mean": 3.254166666666667, + "n": 120, + "num_samples": 1000 + }, + "mean": 3.254166666666667, + "per_seed": { + "1073161778/train0": 3.0, + "1073161778/train1": 1.0, + "1073161778/train2": 2.0, + "1113371261/train0": 7.0, + "1113371261/train1": 3.0, + "1113371261/train2": 7.0, + "1151410346/train0": 6.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 3.0, + "1261930456/train1": 0.0, + "1261930456/train2": 3.0, + "1324133122/train0": 7.0, + "1324133122/train1": 1.0, + "1324133122/train2": 0.0, + "1324642411/train0": 5.0, + "1324642411/train1": 7.0, + "1324642411/train2": 3.0, + "1375492786/train0": 2.0, + "1375492786/train1": 5.0, + "1375492786/train2": 4.0, + "141984615/train0": 1.0, + "141984615/train1": 2.0, + "141984615/train2": 0.0, + "1461069713/train0": 2.0, + "1461069713/train1": 6.0, + "1461069713/train2": 7.0, + "1480856810/train0": 1.0, + "1480856810/train1": 4.0, + "1480856810/train2": 3.0, + "1509980664/train0": 7.0, + "1509980664/train1": 6.0, + "1509980664/train2": 7.0, + "1522272686/train0": 1.0, + "1522272686/train1": 7.0, + "1522272686/train2": 1.0, + "1549324805/train0": 5.0, + "1549324805/train1": 7.0, + "1549324805/train2": 6.0, + "1578990937/train0": 4.0, + "1578990937/train1": 5.0, + "1578990937/train2": 0.0, + "1747178030/train0": 4.0, + "1747178030/train1": 1.0, + "1747178030/train2": 2.0, + "1790188787/train0": 1.0, + "1790188787/train1": 0.0, + "1790188787/train2": 3.0, + "1822210471/train0": 0.0, + "1822210471/train1": 3.0, + "1822210471/train2": 2.0, + "1845489013/train0": 4.0, + "1845489013/train1": 3.0, + "1845489013/train2": 7.0, + "1999022574/train0": 6.0, + "1999022574/train1": 2.0, + "1999022574/train2": 4.0, + "2012211287/train0": 2.0, + "2012211287/train1": 2.0, + "2012211287/train2": 2.0, + "2038594191/train0": 7.0, + "2038594191/train1": 5.0, + "2038594191/train2": 3.0, + "2054899126/train0": 0.5, + "2054899126/train1": 4.0, + "2054899126/train2": 3.5, + "2061635888/train0": 7.0, + "2061635888/train1": 1.0, + "2061635888/train2": 2.0, + "211245842/train0": 1.0, + "211245842/train1": 2.0, + "211245842/train2": 4.0, + "278008524/train0": 6.0, + "278008524/train1": 4.0, + "278008524/train2": 1.0, + "307068879/train0": 0.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 4.0, + "412102563/train1": 4.0, + "412102563/train2": 7.0, + "440577711/train0": 1.0, + "440577711/train1": 3.0, + "440577711/train2": 4.0, + "441046872/train0": 1.0, + "441046872/train1": 0.0, + "441046872/train2": 2.0, + "509430392/train0": 4.0, + "509430392/train1": 2.0, + "509430392/train2": 0.0, + "585904130/train0": 2.0, + "585904130/train1": 4.0, + "585904130/train2": 3.0, + "608016290/train0": 5.0, + "608016290/train1": 1.0, + "608016290/train2": 3.0, + "626486813/train0": 1.0, + "626486813/train1": 5.5, + "626486813/train2": 0.5, + "636680137/train0": 5.0, + "636680137/train1": 1.0, + "636680137/train2": 7.0, + "655884605/train0": 7.0, + "655884605/train1": 3.0, + "655884605/train2": 3.0, + "672328262/train0": 0.0, + "672328262/train1": 6.0, + "672328262/train2": 5.0, + "701667486/train0": 4.0, + "701667486/train1": 2.0, + "701667486/train2": 4.0, + "747260437/train0": 6.0, + "747260437/train1": 0.0, + "747260437/train2": 7.0, + "750448296/train0": 3.6666666666666665, + "750448296/train1": 5.0, + "750448296/train2": 4.333333333333333, + "767747424/train0": 1.75, + "767747424/train1": 2.0, + "767747424/train2": 4.75 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 3.521979797979798, + "low": 2.2528888888888887, + "mean": 2.929252525252525, + "n": 5, + "num_samples": 1000 + }, + "mean": 2.929252525252525, + "per_task": { + "LiftPegUpright-v1": 1.7333333333333334, + "PickCube-v1": 3.0, + "PullCube-v1": 3.453333333333333, + "PushCube-v1": 2.611111111111111, + "StackCube-v1": 3.8484848484848486 + } + }, + "metric": "selected_candidate_index", + "micro": { + "confidence": 0.95, + "high": 3.6458333333333335, + "low": 2.861111111111111, + "mean": 3.25, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.40277777777777773, + "low": 0.24652777777777776, + "mean": 0.3215277777777778, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3215277777777778, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 0.0, + "1073161778/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 1.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 1.0, + "1845489013/train1": 1.0, + "1845489013/train2": 1.0, + "1999022574/train0": 1.0, + "1999022574/train1": 1.0, + "1999022574/train2": 1.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 1.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 1.0, + "412102563/train1": 1.0, + "412102563/train2": 0.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 1.0, + "672328262/train1": 1.0, + "672328262/train2": 1.0, + "701667486/train0": 0.0, + "701667486/train1": 1.0, + "701667486/train2": 1.0, + "747260437/train0": 1.0, + "747260437/train1": 1.0, + "747260437/train2": 1.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.31999999999999995, + "low": 0.11515151515151514, + "mean": 0.23515151515151517, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.23515151515151517, + "per_task": { + "LiftPegUpright-v1": 0.26666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.24242424242424243 + } + }, + "metric": "selected_success", + "micro": { + "confidence": 0.95, + "high": 0.375, + "low": 0.2222222222222222, + "mean": 0.2986111111111111, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_success_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.09444444444444443, + "low": -0.0638888888888889, + "mean": 0.015277777777777777, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.015277777777777777, + "per_seed": { + "1073161778/train0": -1.0, + "1073161778/train1": -1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": -1.0, + "1261930456/train1": -1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 0.0, + "1375492786/train0": -1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.0, + "1480856810/train1": 1.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 1.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 1.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 1.0, + "2038594191/train0": 0.0, + "2038594191/train1": -1.0, + "2038594191/train2": -1.0, + "2054899126/train0": -0.5, + "2054899126/train1": -0.5, + "2054899126/train2": -0.5, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": -1.0, + "440577711/train0": 1.0, + "440577711/train1": 1.0, + "440577711/train2": 0.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 0.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 1.0, + "585904130/train1": 1.0, + "585904130/train2": 1.0, + "608016290/train0": 1.0, + "608016290/train1": 1.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": -1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.10727272727272726, + "low": -0.02278787878787879, + "mean": 0.035272727272727275, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.035272727272727275, + "per_task": { + "LiftPegUpright-v1": 0.06666666666666667, + "PickCube-v1": 0.0, + "PullCube-v1": -0.02666666666666667, + "PushCube-v1": 0.16666666666666666, + "StackCube-v1": -0.030303030303030304 + } + }, + "metric": "selected_success_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.0763888888888889, + "low": -0.0625, + "mean": 0.006944444444444444, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9688077165505902, + "low": 0.6809279332411179, + "mean": 0.8146957620874875, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.8146957620874875, + "per_seed": { + "1073161778/train0": 0.6962459683418274, + "1073161778/train1": 0.5317060947418213, + "1073161778/train2": 2.0, + "1113371261/train0": 0.15842342376708984, + "1113371261/train1": 0.15842342376708984, + "1113371261/train2": 0.15842342376708984, + "1151410346/train0": 0.2260012924671173, + "1151410346/train1": 0.8134199976921082, + "1151410346/train2": 0.2260012924671173, + "1261930456/train0": 0.06716175377368927, + "1261930456/train1": 0.07528233528137207, + "1261930456/train2": 2.0, + "1324133122/train0": 0.09220491349697113, + "1324133122/train1": 0.09220491349697113, + "1324133122/train2": 0.09220491349697113, + "1324642411/train0": 0.17193320393562317, + "1324642411/train1": 0.06962272524833679, + "1324642411/train2": 0.10968391597270966, + "1375492786/train0": 0.008132100105285645, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.10283424705266953, + "141984615/train1": 0.10283424705266953, + "141984615/train2": 0.10283424705266953, + "1461069713/train0": 0.0939030647277832, + "1461069713/train1": 0.0939030647277832, + "1461069713/train2": 0.0939030647277832, + "1480856810/train0": 2.0, + "1480856810/train1": 2.0, + "1480856810/train2": 0.024744659662246704, + "1509980664/train0": 0.8334100246429443, + "1509980664/train1": 2.0, + "1509980664/train2": 0.8334100246429443, + "1522272686/train0": 0.5839591026306152, + "1522272686/train1": 0.5839591026306152, + "1522272686/train2": 0.5839591026306152, + "1549324805/train0": 0.10451483726501465, + "1549324805/train1": 0.10451483726501465, + "1549324805/train2": 0.10451483726501465, + "1578990937/train0": 0.82413649559021, + "1578990937/train1": 0.82413649559021, + "1578990937/train2": 0.82413649559021, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1790188787/train0": 0.06047999858856201, + "1790188787/train1": 0.1146477460861206, + "1790188787/train2": 2.0, + "1822210471/train0": 0.06306678056716919, + "1822210471/train1": 0.07191530615091324, + "1822210471/train2": 0.08819514513015747, + "1845489013/train0": 2.0, + "1845489013/train1": 2.0, + "1845489013/train2": 2.0, + "1999022574/train0": 2.0, + "1999022574/train1": 2.0, + "1999022574/train2": 2.0, + "2012211287/train0": 0.7236053347587585, + "2012211287/train1": 0.7183246612548828, + "2012211287/train2": 2.0, + "2038594191/train0": 2.0, + "2038594191/train1": 0.07674666494131088, + "2038594191/train2": 0.12977690994739532, + "2054899126/train0": 1.0714938417077065, + "2054899126/train1": 1.0660420358181, + "2054899126/train2": 1.0276100430637598, + "2061635888/train0": 0.05225050449371338, + "2061635888/train1": 0.05225050449371338, + "2061635888/train2": 0.05225050449371338, + "211245842/train0": 0.12738418579101562, + "211245842/train1": 0.19090841710567474, + "211245842/train2": 0.19090841710567474, + "278008524/train0": 0.042734989896416664, + "278008524/train1": 0.042734989896416664, + "278008524/train2": 0.042734989896416664, + "307068879/train0": 0.6000905632972717, + "307068879/train1": 0.6000905632972717, + "307068879/train2": 0.6000905632972717, + "412102563/train0": 2.0, + "412102563/train1": 2.0, + "412102563/train2": 0.8126504421234131, + "440577711/train0": 2.0, + "440577711/train1": 2.0, + "440577711/train2": 0.12224134802818298, + "441046872/train0": 0.6999551653862, + "441046872/train1": 0.6999551653862, + "441046872/train2": 0.6999551653862, + "509430392/train0": 6.109476089477539e-07, + "509430392/train1": 1.519918441772461e-06, + "509430392/train2": 2.682209014892578e-07, + "585904130/train0": 2.0, + "585904130/train1": 2.0, + "585904130/train2": 2.0, + "608016290/train0": 2.0, + "608016290/train1": 2.0, + "608016290/train2": 0.725701630115509, + "626486813/train0": 0.00509834272088483, + "626486813/train1": 0.006621529464609921, + "626486813/train2": 0.00509834272088483, + "636680137/train0": 0.08313504606485367, + "636680137/train1": 0.08313504606485367, + "636680137/train2": 0.08313504606485367, + "655884605/train0": 0.16300667822360992, + "655884605/train1": 0.16300667822360992, + "655884605/train2": 0.16300667822360992, + "672328262/train0": 2.0, + "672328262/train1": 2.0, + "672328262/train2": 2.0, + "701667486/train0": 0.18805170059204102, + "701667486/train1": 2.0, + "701667486/train2": 2.0, + "747260437/train0": 2.0, + "747260437/train1": 2.0, + "747260437/train2": 2.0, + "750448296/train0": 0.7452354828516642, + "750448296/train1": 0.13002302000919977, + "750448296/train2": 0.10690136750539143, + "767747424/train0": 0.5375678082928061, + "767747424/train1": 0.5778386716265231, + "767747424/train2": 0.5891513926908374 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8997010686323048, + "low": 0.6416583168216501, + "mean": 0.7544153995025112, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.7544153995025112, + "per_task": { + "LiftPegUpright-v1": 1.0197147250175476, + "PickCube-v1": 0.5839591026306152, + "PullCube-v1": 0.7196805840544402, + "PushCube-v1": 0.7119888936479887, + "StackCube-v1": 0.7367336921619646 + } + }, + "metric": "selected_utility", + "micro": { + "confidence": 0.95, + "high": 0.8855680860360735, + "low": 0.6056447847853027, + "mean": 0.7510531521822688, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selected_utility_gain_over_base": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.16246743871597574, + "low": -0.10955652395011081, + "mean": 0.02457769978969332, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.02457769978969332, + "per_seed": { + "1073161778/train0": -1.3037540316581726, + "1073161778/train1": -1.4682939052581787, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.5874187052249908, + "1151410346/train2": 0.0, + "1261930456/train0": -1.9328382462263107, + "1261930456/train1": -1.924717664718628, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.06546267867088318, + "1324642411/train1": -0.0368478000164032, + "1324642411/train2": 0.0032133907079696655, + "1375492786/train0": -1.9918678998947144, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 1.9129659235477448, + "1480856810/train1": 1.9129659235477448, + "1480856810/train2": -0.062289416790008545, + "1509980664/train0": 0.0, + "1509980664/train1": 1.1665899753570557, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": -0.064287468791008, + "1790188787/train1": -0.010119721293449402, + "1790188787/train2": 1.87523253262043, + "1822210471/train0": -0.008848525583744049, + "1822210471/train1": 0.0, + "1822210471/train2": 0.016279838979244232, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": -0.005280673503875732, + "2012211287/train2": 1.2763946652412415, + "2038594191/train0": 0.0, + "2038594191/train1": -1.9232533350586891, + "2038594191/train2": -1.8702230900526047, + "2054899126/train0": -0.9285061582922935, + "2054899126/train1": -0.9339579641819, + "2054899126/train2": -0.9723899569362402, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": -0.06352423131465912, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": -1.187349557876587, + "440577711/train0": 1.9638472199440002, + "440577711/train1": 1.9638472199440002, + "440577711/train2": 0.08608856797218323, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 3.8743019104003906e-07, + "509430392/train1": 1.296401023864746e-06, + "509430392/train2": 4.470348358154297e-08, + "585904130/train0": 1.8182844519615173, + "585904130/train1": 1.8182844519615173, + "585904130/train2": 1.8182844519615173, + "608016290/train0": 1.274298369884491, + "608016290/train1": 1.274298369884491, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0015231867437250912, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": -1.811948299407959, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.6276423335075378, + "750448296/train1": 0.012429870665073395, + "750448296/train2": -0.010691781838734945, + "767747424/train0": -0.035631533712148666, + "767747424/train1": 0.004639329621568322, + "767747424/train2": 0.01595205068588257 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.19462169074919075, + "low": -0.03806035581327072, + "mean": 0.05885560232849622, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.05885560232849622, + "per_task": { + "LiftPegUpright-v1": 0.06984418630599976, + "PickCube-v1": 0.0, + "PullCube-v1": -0.05548987716902047, + "PushCube-v1": 0.31958471486965817, + "StackCube-v1": -0.03966101236415632 + } + }, + "metric": "selected_utility_gain_over_base", + "micro": { + "confidence": 0.95, + "high": 0.12995398117669132, + "low": -0.11376581491074628, + "mean": 0.00923356573993159, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "selector_regret": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.48948979464362813, + "low": 0.2585044706822373, + "mean": 0.3702363975384893, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3702363975384893, + "per_seed": { + "1073161778/train0": 1.3037540316581726, + "1073161778/train1": 0.1754935383796692, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.02250880002975464, + "1113371261/train2": 0.036848247051239014, + "1151410346/train0": 0.6403508484363556, + "1151410346/train1": 0.00047194957733154297, + "1151410346/train2": 0.0, + "1261930456/train0": 1.9328382462263107, + "1261930456/train1": 1.924717664718628, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.04025374352931976, + "1324133122/train2": 0.10070674121379852, + "1324642411/train0": 0.0, + "1324642411/train1": 0.1140681803226471, + "1324642411/train2": 1.8903160840272903, + "1375492786/train0": 1.9918678998947144, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.005800329148769379, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.9752553403377533, + "1509980664/train0": 1.1665899753570557, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.09440667927265167, + "1549324805/train2": 0.0, + "1578990937/train0": 1.17586350440979, + "1578990937/train1": 1.17586350440979, + "1578990937/train2": 1.17586350440979, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.037439242005348206, + "1790188787/train1": 0.015666961669921875, + "1790188787/train2": 0.0, + "1822210471/train0": 1.9369332194328308, + "1822210471/train1": 1.9280846938490868, + "1822210471/train2": 1.9118048548698425, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.2763946652412415, + "2012211287/train1": 1.2816753387451172, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 1.9232533350586891, + "2038594191/train2": 1.8702230900526047, + "2054899126/train0": 0.005789309740066528, + "2054899126/train1": 0.003638841211795807, + "2054899126/train2": 0.017916223034262657, + "2061635888/train0": 0.04574880003929138, + "2061635888/train1": 0.02355286478996277, + "2061635888/train2": 0.05880081653594971, + "211245842/train0": 1.8726158142089844, + "211245842/train1": 0.02204875648021698, + "211245842/train2": 0.02432851493358612, + "278008524/train0": 0.985974382609129, + "278008524/train1": 0.9949396047741175, + "278008524/train2": 1.0932139214128256, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.187349557876587, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 1.877758651971817, + "441046872/train0": 0.0, + "441046872/train1": 0.023403704166412354, + "441046872/train2": 0.013073742389678955, + "509430392/train0": 1.999999389052391, + "509430392/train1": 6.012618541717529e-05, + "509430392/train2": 5.155801773071289e-06, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.038800019945483655, + "626486813/train1": 0.023933748132549226, + "626486813/train2": 0.03346832672832534, + "636680137/train0": 0.005126260221004486, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0021464377641677856, + "655884605/train2": 0.05146662890911102, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.811948299407959, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.024625877539316814, + "750448296/train1": 1.240480254093806, + "750448296/train2": 1.2574311296145122, + "767747424/train0": 0.5135916564613581, + "767747424/train1": 0.025535901775583625, + "767747424/train2": 0.024280773475766182 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5209456862919033, + "low": 0.12891305829539443, + "mean": 0.32388832561028275, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.32388832561028275, + "per_task": { + "LiftPegUpright-v1": 0.27158633470535276, + "PickCube-v1": 0.0, + "PullCube-v1": 0.37178792303428054, + "PushCube-v1": 0.6537847245732943, + "StackCube-v1": 0.3222826457384861 + } + }, + "metric": "selector_regret", + "micro": { + "confidence": 0.95, + "high": 0.5048640615819345, + "low": 0.26330452243645497, + "mean": 0.3775093166655602, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_selector_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.27777777777777773, + "low": 0.14444444444444443, + "mean": 0.20902777777777778, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.20902777777777778, + "per_seed": { + "1073161778/train0": 1.0, + "1073161778/train1": 0.0, + "1073161778/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1151410346/train0": 0.0, + "1151410346/train1": 0.0, + "1151410346/train2": 0.0, + "1261930456/train0": 1.0, + "1261930456/train1": 1.0, + "1261930456/train2": 0.0, + "1324133122/train0": 0.0, + "1324133122/train1": 0.0, + "1324133122/train2": 0.0, + "1324642411/train0": 0.0, + "1324642411/train1": 0.0, + "1324642411/train2": 1.0, + "1375492786/train0": 1.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 1.0, + "1509980664/train0": 1.0, + "1509980664/train1": 0.0, + "1509980664/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 1.0, + "1578990937/train1": 1.0, + "1578990937/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 0.0, + "1790188787/train1": 0.0, + "1790188787/train2": 0.0, + "1822210471/train0": 1.0, + "1822210471/train1": 1.0, + "1822210471/train2": 1.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 1.0, + "2012211287/train1": 1.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 1.0, + "2038594191/train2": 1.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "2061635888/train0": 0.0, + "2061635888/train1": 0.0, + "2061635888/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "307068879/train0": 0.0, + "307068879/train1": 0.0, + "307068879/train2": 0.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 1.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 1.0, + "441046872/train0": 0.0, + "441046872/train1": 0.0, + "441046872/train2": 0.0, + "509430392/train0": 1.0, + "509430392/train1": 0.0, + "509430392/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 0.0, + "655884605/train1": 0.0, + "655884605/train2": 0.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 1.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.6666666666666666, + "750448296/train2": 0.6666666666666666, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.27975757575757576, + "low": 0.07975757575757576, + "mean": 0.18642424242424244, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.18642424242424244, + "per_task": { + "LiftPegUpright-v1": 0.2, + "PickCube-v1": 0.0, + "PullCube-v1": 0.18666666666666668, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.21212121212121213 + } + }, + "metric": "success_selector_gap", + "micro": { + "confidence": 0.95, + "high": 0.2777777777777778, + "low": 0.14583333333333334, + "mean": 0.20833333333333334, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "success_support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39861111111111114, + "low": 0.2375, + "mean": 0.3194444444444445, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3194444444444445, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.0, + "1073161778/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1151410346/train0": 1.0, + "1151410346/train1": 1.0, + "1151410346/train2": 1.0, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.0, + "1324133122/train1": 1.0, + "1324133122/train2": 1.0, + "1324642411/train0": 1.0, + "1324642411/train1": 1.0, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1461069713/train0": 0.0, + "1461069713/train1": 0.0, + "1461069713/train2": 0.0, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.0, + "1790188787/train1": 1.0, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.5, + "2054899126/train1": 0.5, + "2054899126/train2": 0.5, + "2061635888/train0": 1.0, + "2061635888/train1": 1.0, + "2061635888/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "307068879/train0": 1.0, + "307068879/train1": 1.0, + "307068879/train2": 1.0, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.0, + "441046872/train1": 1.0, + "441046872/train2": 1.0, + "509430392/train0": 0.0, + "509430392/train1": 1.0, + "509430392/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "655884605/train0": 1.0, + "655884605/train1": 1.0, + "655884605/train2": 1.0, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.0, + "747260437/train1": 0.0, + "747260437/train2": 0.0, + "750448296/train0": 0.3333333333333333, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.25, + "767747424/train2": 0.25 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7757575757575758, + "low": 0.23733333333333334, + "mean": 0.501090909090909, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.501090909090909, + "per_task": { + "LiftPegUpright-v1": 0.5333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.09333333333333334, + "PushCube-v1": 0.3333333333333333, + "StackCube-v1": 0.5454545454545454 + } + }, + "metric": "success_support_gap", + "micro": { + "confidence": 0.95, + "high": 0.3611111111111111, + "low": 0.2222222222222222, + "mean": 0.2916666666666667, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "support_gap": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7096626987680793, + "low": 0.43689682670713714, + "mean": 0.5784295155162302, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.5784295155162302, + "per_seed": { + "1073161778/train0": 0.0, + "1073161778/train1": 1.2928003668785095, + "1073161778/train2": 0.0, + "1113371261/train0": 1.897251382470131, + "1113371261/train1": 1.8190677762031555, + "1113371261/train2": 1.8047283291816711, + "1151410346/train0": 1.133647859096527, + "1151410346/train1": 1.1861080527305603, + "1151410346/train2": 1.7808725833892822, + "1261930456/train0": 0.0, + "1261930456/train1": 0.0, + "1261930456/train2": 0.0, + "1324133122/train0": 1.9450107961893082, + "1324133122/train1": 1.867541342973709, + "1324133122/train2": 1.8070883452892303, + "1324642411/train0": 1.8280667960643768, + "1324642411/train1": 1.8163090944290161, + "1324642411/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 1.9174813032150269, + "1375492786/train2": 0.0, + "141984615/train0": 0.27848575497046113, + "141984615/train1": 0.24876614287495613, + "141984615/train2": 0.2555490341037512, + "1461069713/train0": 0.0991440862417221, + "1461069713/train1": 0.14275743998587132, + "1461069713/train2": 0.059639059007167816, + "1480856810/train0": 0.0, + "1480856810/train1": 0.0, + "1480856810/train2": 0.0, + "1509980664/train0": 0.0, + "1509980664/train1": 0.0, + "1509980664/train2": 1.1837782859802246, + "1522272686/train0": 1.8167466223239899, + "1522272686/train1": 1.9298561662435532, + "1522272686/train2": 1.8506269305944443, + "1549324805/train0": 0.255916815251112, + "1549324805/train1": 0.09853456914424896, + "1549324805/train2": 0.20652779191732407, + "1578990937/train0": 0.0, + "1578990937/train1": 0.0, + "1578990937/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1790188787/train0": 1.9020807594060898, + "1790188787/train1": 1.8696852922439575, + "1790188787/train2": 0.0, + "1822210471/train0": 0.0, + "1822210471/train1": 0.0, + "1822210471/train2": 0.0, + "1845489013/train0": 0.0, + "1845489013/train1": 0.0, + "1845489013/train2": 0.0, + "1999022574/train0": 0.0, + "1999022574/train1": 0.0, + "1999022574/train2": 0.0, + "2012211287/train0": 0.0, + "2012211287/train1": 0.0, + "2012211287/train2": 0.0, + "2038594191/train0": 0.0, + "2038594191/train1": 0.0, + "2038594191/train2": 0.0, + "2054899126/train0": 0.922716848552227, + "2054899126/train1": 0.9303191229701042, + "2054899126/train2": 0.9544737339019775, + "2061635888/train0": 1.9020006954669952, + "2061635888/train1": 1.9241966307163239, + "2061635888/train2": 1.888948678970337, + "211245842/train0": 0.0, + "211245842/train1": 1.7870428264141083, + "211245842/train2": 1.7847630679607391, + "278008524/train0": 0.08519926853477955, + "278008524/train1": 0.07623404636979103, + "278008524/train2": 0.0, + "307068879/train0": 1.7884424477815628, + "307068879/train1": 1.4123926758766174, + "307068879/train2": 1.7727213501930237, + "412102563/train0": 0.0, + "412102563/train1": 0.0, + "412102563/train2": 0.0, + "440577711/train0": 0.0, + "440577711/train1": 0.0, + "440577711/train2": 0.0, + "441046872/train0": 1.4851974844932556, + "441046872/train1": 1.2766411304473877, + "441046872/train2": 1.286971092224121, + "509430392/train0": 0.0, + "509430392/train1": 1.999938353896141, + "509430392/train2": 1.9999945759773254, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "608016290/train0": 0.0, + "608016290/train1": 0.0, + "608016290/train2": 1.3174219727516174, + "626486813/train0": 0.12241578102111816, + "626486813/train1": 0.13502710359171033, + "626486813/train2": 0.1270157117396593, + "636680137/train0": 0.0823030173778534, + "636680137/train1": 0.10315313935279846, + "636680137/train2": 0.09491270035505295, + "655884605/train0": 1.8626713156700134, + "655884605/train1": 1.8348468840122223, + "655884605/train2": 1.785526692867279, + "672328262/train0": 0.0, + "672328262/train1": 0.0, + "672328262/train2": 0.0, + "701667486/train0": 0.0, + "701667486/train1": 0.0, + "701667486/train2": 0.0, + "747260437/train0": 0.17026761174201965, + "747260437/train1": 0.22598901391029358, + "747260437/train2": 0.15356136858463287, + "750448296/train0": 0.6553361676633358, + "750448296/train1": 0.03749322146177292, + "750448296/train2": 0.05815017223358154, + "767747424/train0": 0.06562284985557199, + "767747424/train1": 0.5129444571211934, + "767747424/train2": 0.49461987148970366 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.4353523810704547, + "low": 0.47464505295703807, + "mean": 0.8912108774023751, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.8912108774023751, + "per_task": { + "LiftPegUpright-v1": 0.7755059013764064, + "PickCube-v1": 1.8657432397206624, + "PullCube-v1": 0.23527305972451965, + "PushCube-v1": 0.634226381778717, + "StackCube-v1": 0.9453058044115702 + } + }, + "metric": "support_gap", + "micro": { + "confidence": 0.95, + "high": 0.6574370106811531, + "low": 0.4129596278103741, + "mean": 0.5381004453940679, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + }, + "tau": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3440815679198535, + "low": 0.3440815679198535, + "mean": 0.3440815679198535, + "n": 120, + "num_samples": 1000 + }, + "mean": 0.3440815679198535, + "per_seed": { + "1073161778/train0": 0.34408156791985406, + "1073161778/train1": 0.34408156791985406, + "1073161778/train2": 0.34408156791985406, + "1113371261/train0": 0.34408156791985406, + "1113371261/train1": 0.34408156791985406, + "1113371261/train2": 0.34408156791985406, + "1151410346/train0": 0.34408156791985406, + "1151410346/train1": 0.34408156791985406, + "1151410346/train2": 0.34408156791985406, + "1261930456/train0": 0.34408156791985406, + "1261930456/train1": 0.34408156791985406, + "1261930456/train2": 0.34408156791985406, + "1324133122/train0": 0.34408156791985406, + "1324133122/train1": 0.34408156791985406, + "1324133122/train2": 0.34408156791985406, + "1324642411/train0": 0.34408156791985406, + "1324642411/train1": 0.34408156791985406, + "1324642411/train2": 0.34408156791985406, + "1375492786/train0": 0.34408156791985406, + "1375492786/train1": 0.34408156791985406, + "1375492786/train2": 0.34408156791985406, + "141984615/train0": 0.34408156791985406, + "141984615/train1": 0.34408156791985406, + "141984615/train2": 0.34408156791985406, + "1461069713/train0": 0.34408156791985406, + "1461069713/train1": 0.34408156791985406, + "1461069713/train2": 0.34408156791985406, + "1480856810/train0": 0.34408156791985406, + "1480856810/train1": 0.34408156791985406, + "1480856810/train2": 0.34408156791985406, + "1509980664/train0": 0.34408156791985406, + "1509980664/train1": 0.34408156791985406, + "1509980664/train2": 0.34408156791985406, + "1522272686/train0": 0.34408156791985406, + "1522272686/train1": 0.34408156791985406, + "1522272686/train2": 0.34408156791985406, + "1549324805/train0": 0.34408156791985406, + "1549324805/train1": 0.34408156791985406, + "1549324805/train2": 0.34408156791985406, + "1578990937/train0": 0.34408156791985406, + "1578990937/train1": 0.34408156791985406, + "1578990937/train2": 0.34408156791985406, + "1747178030/train0": 0.34408156791985406, + "1747178030/train1": 0.34408156791985406, + "1747178030/train2": 0.34408156791985406, + "1790188787/train0": 0.34408156791985406, + "1790188787/train1": 0.34408156791985406, + "1790188787/train2": 0.34408156791985406, + "1822210471/train0": 0.34408156791985406, + "1822210471/train1": 0.34408156791985406, + "1822210471/train2": 0.34408156791985406, + "1845489013/train0": 0.34408156791985406, + "1845489013/train1": 0.34408156791985406, + "1845489013/train2": 0.34408156791985406, + "1999022574/train0": 0.34408156791985406, + "1999022574/train1": 0.34408156791985406, + "1999022574/train2": 0.34408156791985406, + "2012211287/train0": 0.34408156791985406, + "2012211287/train1": 0.34408156791985406, + "2012211287/train2": 0.34408156791985406, + "2038594191/train0": 0.34408156791985406, + "2038594191/train1": 0.34408156791985406, + "2038594191/train2": 0.34408156791985406, + "2054899126/train0": 0.34408156791985406, + "2054899126/train1": 0.34408156791985406, + "2054899126/train2": 0.34408156791985406, + "2061635888/train0": 0.34408156791985406, + "2061635888/train1": 0.34408156791985406, + "2061635888/train2": 0.34408156791985406, + "211245842/train0": 0.34408156791985406, + "211245842/train1": 0.34408156791985406, + "211245842/train2": 0.34408156791985406, + "278008524/train0": 0.34408156791985406, + "278008524/train1": 0.34408156791985406, + "278008524/train2": 0.34408156791985406, + "307068879/train0": 0.34408156791985406, + "307068879/train1": 0.34408156791985406, + "307068879/train2": 0.34408156791985406, + "412102563/train0": 0.34408156791985406, + "412102563/train1": 0.34408156791985406, + "412102563/train2": 0.34408156791985406, + "440577711/train0": 0.34408156791985406, + "440577711/train1": 0.34408156791985406, + "440577711/train2": 0.34408156791985406, + "441046872/train0": 0.34408156791985406, + "441046872/train1": 0.34408156791985406, + "441046872/train2": 0.34408156791985406, + "509430392/train0": 0.34408156791985406, + "509430392/train1": 0.34408156791985406, + "509430392/train2": 0.34408156791985406, + "585904130/train0": 0.34408156791985406, + "585904130/train1": 0.34408156791985406, + "585904130/train2": 0.34408156791985406, + "608016290/train0": 0.34408156791985406, + "608016290/train1": 0.34408156791985406, + "608016290/train2": 0.34408156791985406, + "626486813/train0": 0.34408156791985406, + "626486813/train1": 0.34408156791985406, + "626486813/train2": 0.34408156791985406, + "636680137/train0": 0.34408156791985406, + "636680137/train1": 0.34408156791985406, + "636680137/train2": 0.34408156791985406, + "655884605/train0": 0.34408156791985406, + "655884605/train1": 0.34408156791985406, + "655884605/train2": 0.34408156791985406, + "672328262/train0": 0.34408156791985406, + "672328262/train1": 0.34408156791985406, + "672328262/train2": 0.34408156791985406, + "701667486/train0": 0.34408156791985406, + "701667486/train1": 0.34408156791985406, + "701667486/train2": 0.34408156791985406, + "747260437/train0": 0.34408156791985406, + "747260437/train1": 0.34408156791985406, + "747260437/train2": 0.34408156791985406, + "750448296/train0": 0.34408156791985406, + "750448296/train1": 0.34408156791985406, + "750448296/train2": 0.34408156791985406, + "767747424/train0": 0.34408156791985406, + "767747424/train1": 0.34408156791985406, + "767747424/train2": 0.34408156791985406 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3440815679198542, + "low": 0.34408156791985406, + "mean": 0.3440815679198541, + "n": 5, + "num_samples": 1000 + }, + "mean": 0.3440815679198541, + "per_task": { + "LiftPegUpright-v1": 0.34408156791985406, + "PickCube-v1": 0.34408156791985406, + "PullCube-v1": 0.3440815679198542, + "PushCube-v1": 0.3440815679198541, + "StackCube-v1": 0.34408156791985417 + } + }, + "metric": "tau", + "micro": { + "confidence": 0.95, + "high": 0.34408156791985306, + "low": 0.34408156791985306, + "mean": 0.34408156791985306, + "n": 144, + "num_samples": 1000 + }, + "n": 144 + } + }, + "target": "success", + "tau": 0.34408156791985406 +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics_by_seed.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..0c10a9ca375ff210c45df309e7ee422e072f764c --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics_by_seed.json @@ -0,0 +1,2762 @@ +{ + "1073161778/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5047786612880557, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.6962459683418274, + "selected_utility_gain_over_base": -1.3037540316581726, + "selector_regret": 1.3037540316581726, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1073161778/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5323396378653275, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7071996331214905, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.5317060947418213, + "selected_utility_gain_over_base": -1.4682939052581787, + "selector_regret": 0.1754935383796692, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2928003668785095, + "tau": 0.34408156791985406 + }, + "1073161778/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5132737468890538, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1113371261/train0": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.20380714554880489, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10274861752986908, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.897251382470131, + "tau": 0.34408156791985406 + }, + "1113371261/train1": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.216136141459529, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18093222379684448, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02250880002975464, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8190677762031555, + "tau": 0.34408156791985406 + }, + "1113371261/train2": { + "base_success": 0.0, + "base_utility": 0.15842342376708984, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1818611172967026, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19527167081832886, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.15842342376708984, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.036848247051239014, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8047283291816711, + "tau": 0.34408156791985406 + }, + "1151410346/train0": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16380705661970563, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8663521409034729, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.6403508484363556, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.133647859096527, + "tau": 0.34408156791985406 + }, + "1151410346/train1": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3807159751095167, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8138919472694397, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8134199976921082, + "selected_utility_gain_over_base": 0.5874187052249908, + "selector_regret": 0.00047194957733154297, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1861080527305603, + "tau": 0.34408156791985406 + }, + "1151410346/train2": { + "base_success": 0.0, + "base_utility": 0.2260012924671173, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.22275232189021582, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21912741661071777, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.2260012924671173, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7808725833892822, + "tau": 0.34408156791985406 + }, + "1261930456/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8068754349598813, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.06716175377368927, + "selected_utility_gain_over_base": -1.9328382462263107, + "selector_regret": 1.9328382462263107, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1261930456/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8314472386685839, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07528233528137207, + "selected_utility_gain_over_base": -1.924717664718628, + "selector_regret": 1.924717664718628, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1261930456/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5203682065298446, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1324133122/train0": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.0267850715243588, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.054989203810691833, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9450107961893082, + "tau": 0.34408156791985406 + }, + "1324133122/train1": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.166222195833595, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1324586570262909, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04025374352931976, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.867541342973709, + "tau": 0.34408156791985406 + }, + "1324133122/train2": { + "base_success": 0.0, + "base_utility": 0.09220491349697113, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1895440816104003, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19291165471076965, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.09220491349697113, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.10070674121379852, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8070883452892303, + "tau": 0.34408156791985406 + }, + "1324642411/train0": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6157618702014627, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.17193320393562317, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.17193320393562317, + "selected_utility_gain_over_base": 0.06546267867088318, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8280667960643768, + "tau": 0.34408156791985406 + }, + "1324642411/train1": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7200821909843477, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1836909055709839, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06962272524833679, + "selected_utility_gain_over_base": -0.0368478000164032, + "selector_regret": 0.1140681803226471, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8163090944290161, + "tau": 0.34408156791985406 + }, + "1324642411/train2": { + "base_success": 0.0, + "base_utility": 0.10647052526473999, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8093176408240023, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10968391597270966, + "selected_utility_gain_over_base": 0.0032133907079696655, + "selector_regret": 1.8903160840272903, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5723807687992665, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.008132100105285645, + "selected_utility_gain_over_base": -1.9918678998947144, + "selector_regret": 1.9918678998947144, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.09015312457958802, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08251869678497314, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9174813032150269, + "tau": 0.34408156791985406 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.15985879284842153, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.06373241744268668, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.007291515823453665, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.27848575497046113, + "tau": 0.34408156791985406 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.20426126222013247, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.037011127918958664, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.24876614287495613, + "tau": 0.34408156791985406 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.10283424705266953, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2857772707939148, + "outcome_ptr": 0.0, + "predicted_margin": 0.18128468401517966, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030228236690163612, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10283424705266953, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.2555490341037512, + "tau": 0.34408156791985406 + }, + "1461069713/train0": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.07014763295619539, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06019836664199829, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0991440862417221, + "tau": 0.34408156791985406 + }, + "1461069713/train1": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 0.0, + "predicted_margin": 0.11660139608054601, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.016585012897849083, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.14275743998587132, + "tau": 0.34408156791985406 + }, + "1461069713/train2": { + "base_success": 0.0, + "base_utility": 0.0939030647277832, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.1593424528837204, + "outcome_ptr": 1.0, + "predicted_margin": 0.14708579808151867, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09970339387655258, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.0939030647277832, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005800329148769379, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.059639059007167816, + "tau": 0.34408156791985406 + }, + "1480856810/train0": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.6460010381991991, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1480856810/train1": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.6031466837579302, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9129659235477448, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1480856810/train2": { + "base_success": 0.0, + "base_utility": 0.08703407645225525, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.20948036015033722, + "outcome_ptr": 1.0, + "predicted_margin": 0.5623456840603507, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.024744659662246704, + "selected_utility_gain_over_base": -0.062289416790008545, + "selector_regret": 1.9752553403377533, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1509980664/train0": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2711783741938117, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.1665899753570557, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1509980664/train1": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6271541093649009, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.1665899753570557, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1509980664/train2": { + "base_success": 0.0, + "base_utility": 0.8334100246429443, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3204171162126431, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.8162217140197754, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.8334100246429443, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.1837782859802246, + "tau": 0.34408156791985406 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.028797013011872402, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.18325337767601013, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8167466223239899, + "tau": 0.34408156791985406 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.04289956654184263, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07014383375644684, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9298561662435532, + "tau": 0.34408156791985406 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.036313152627768894, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.14937306940555573, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8506269305944443, + "tau": 0.34408156791985406 + }, + "1549324805/train0": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.11816772500779864, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0415392704308033, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.255916815251112, + "tau": 0.34408156791985406 + }, + "1549324805/train1": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 1.0, + "predicted_margin": 0.12623384749452976, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.19892151653766632, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.09440667927265167, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09853456914424896, + "tau": 0.34408156791985406 + }, + "1549324805/train2": { + "base_success": 0.0, + "base_utility": 0.10451483726501465, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2974560856819153, + "outcome_ptr": 0.0, + "predicted_margin": 0.08943732805519664, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09092829376459122, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10451483726501465, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.20652779191732407, + "tau": 0.34408156791985406 + }, + "1578990937/train0": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1638611043693725, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1578990937/train1": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15477671498906614, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1578990937/train2": { + "base_success": 0.0, + "base_utility": 0.82413649559021, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.09931026220178542, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.82413649559021, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.17586350440979, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1747178030/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7924670056628417, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1747178030/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8683209874745884, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1747178030/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6084844687781336, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1790188787/train0": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8362820060696806, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09791924059391022, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06047999858856201, + "selected_utility_gain_over_base": -0.064287468791008, + "selector_regret": 0.037439242005348206, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020807594060898, + "tau": 0.34408156791985406 + }, + "1790188787/train1": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7982925168010331, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13031470775604248, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.1146477460861206, + "selected_utility_gain_over_base": -0.010119721293449402, + "selector_regret": 0.015666961669921875, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8696852922439575, + "tau": 0.34408156791985406 + }, + "1790188787/train2": { + "base_success": 0.0, + "base_utility": 0.12476746737957001, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.840936591096665, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.87523253262043, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1822210471/train0": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6455405096871031, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.06306678056716919, + "selected_utility_gain_over_base": -0.008848525583744049, + "selector_regret": 1.9369332194328308, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1822210471/train1": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3054049598551906, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.07191530615091324, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.9280846938490868, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1822210471/train2": { + "base_success": 0.0, + "base_utility": 0.07191530615091324, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.521098644476326, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08819514513015747, + "selected_utility_gain_over_base": 0.016279838979244232, + "selector_regret": 1.9118048548698425, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1845489013/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8344191292729335, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1845489013/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.8748628344239395, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1845489013/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7678061787371938, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1999022574/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5365982119223281, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1999022574/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6265576940062217, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "1999022574/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.29912216229597205, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2012211287/train0": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.28601672596330097, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7236053347587585, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.2763946652412415, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2012211287/train1": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5007199064566366, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.7183246612548828, + "selected_utility_gain_over_base": -0.005280673503875732, + "selector_regret": 1.2816753387451172, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2012211287/train2": { + "base_success": 0.0, + "base_utility": 0.7236053347587585, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.448502256643165, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.2763946652412415, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2038594191/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.30175972912544086, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2038594191/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3478707570137989, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.07674666494131088, + "selected_utility_gain_over_base": -1.9232533350586891, + "selector_regret": 1.9232533350586891, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2038594191/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6956523320904741, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.12977690994739532, + "selected_utility_gain_over_base": -1.8702230900526047, + "selector_regret": 1.8702230900526047, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.40038765069149596, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.077283151447773, + "selected_candidate_index": 0.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0714938417077065, + "selected_utility_gain_over_base": -0.9285061582922935, + "selector_regret": 0.005789309740066528, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.922716848552227, + "tau": 0.34408156791985406 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5251440689575144, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0696808770298958, + "selected_candidate_index": 4.0, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0660420358181, + "selected_utility_gain_over_base": -0.9339579641819, + "selector_regret": 0.003638841211795807, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9303191229701042, + "tau": 0.34408156791985406 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.334834464710266, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0455262660980225, + "selected_candidate_index": 3.5, + "selected_success": 0.5, + "selected_success_gain_over_base": -0.5, + "selected_utility": 1.0276100430637598, + "selected_utility_gain_over_base": -0.9723899569362402, + "selector_regret": 0.017916223034262657, + "success_selector_gap": 0.0, + "success_support_gap": 0.5, + "support_gap": 0.9544737339019775, + "tau": 0.34408156791985406 + }, + "2061635888/train0": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.25347796342569456, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.09799930453300476, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.04574880003929138, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9020006954669952, + "tau": 0.34408156791985406 + }, + "2061635888/train1": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2576974397440808, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.07580336928367615, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02355286478996277, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9241966307163239, + "tau": 0.34408156791985406 + }, + "2061635888/train2": { + "base_success": 0.0, + "base_utility": 0.05225050449371338, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.14461486871684395, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.11105132102966309, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.05225050449371338, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05880081653594971, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.888948678970337, + "tau": 0.34408156791985406 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.3785855718496198, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12738418579101562, + "selected_utility_gain_over_base": -0.06352423131465912, + "selector_regret": 1.8726158142089844, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.15046609148838602, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21295717358589172, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02204875648021698, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7870428264141083, + "tau": 0.34408156791985406 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.19090841710567474, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16061195744478388, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21523693203926086, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.19090841710567474, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.02432851493358612, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7847630679607391, + "tau": 0.34408156791985406 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.14581513472810403, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0287093725055456, + "selected_candidate_index": 6.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.985974382609129, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.08519926853477955, + "tau": 0.34408156791985406 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.12964633215572155, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0376745946705341, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.9949396047741175, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.07623404636979103, + "tau": 0.34408156791985406 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.042734989896416664, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.5, + "hidden_chart_oracle_utility": 1.1139086410403252, + "outcome_ptr": 1.0, + "predicted_margin": 0.11421197996697016, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.1359489113092422, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.042734989896416664, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 1.0932139214128256, + "success_selector_gap": 0.5, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "307068879/train0": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1887491123406104, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.2115575522184372, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7884424477815628, + "tau": 0.34408156791985406 + }, + "307068879/train1": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1074108711810591, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5876073241233826, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4123926758766174, + "tau": 0.34408156791985406 + }, + "307068879/train2": { + "base_success": 0.0, + "base_utility": 0.6000905632972717, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.16243232875050095, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.22727864980697632, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6000905632972717, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.7727213501930237, + "tau": 0.34408156791985406 + }, + "412102563/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.7010170953685468, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "412102563/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5698137543063153, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "412102563/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.38545147035771476, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.8126504421234131, + "selected_utility_gain_over_base": -1.187349557876587, + "selector_regret": 1.187349557876587, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "440577711/train0": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7826205556130074, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "440577711/train1": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.7916612246735112, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.9638472199440002, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "440577711/train2": { + "base_success": 0.0, + "base_utility": 0.036152780055999756, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8330203790971261, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.12224134802818298, + "selected_utility_gain_over_base": 0.08608856797218323, + "selector_regret": 1.877758651971817, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "441046872/train0": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.1369652832639738, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.5148025155067444, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.4851974844932556, + "tau": 0.34408156791985406 + }, + "441046872/train1": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.1381008153886979, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7233588695526123, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.023403704166412354, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.2766411304473877, + "tau": 0.34408156791985406 + }, + "441046872/train2": { + "base_success": 0.0, + "base_utility": 0.6999551653862, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.16102481521823614, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.7130289077758789, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.6999551653862, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.013073742389678955, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.286971092224121, + "tau": 0.34408156791985406 + }, + "509430392/train0": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8694829408465863, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 6.109476089477539e-07, + "selected_utility_gain_over_base": 3.8743019104003906e-07, + "selector_regret": 1.999999389052391, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "509430392/train1": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.8876110106518329, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 6.164610385894775e-05, + "selected_candidate_index": 2.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 1.519918441772461e-06, + "selected_utility_gain_over_base": 1.296401023864746e-06, + "selector_regret": 6.012618541717529e-05, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.999938353896141, + "tau": 0.34408156791985406 + }, + "509430392/train2": { + "base_success": 0.0, + "base_utility": 2.2351741790771484e-07, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.877751694992998, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 5.424022674560547e-06, + "selected_candidate_index": 0.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.682209014892578e-07, + "selected_utility_gain_over_base": 4.470348358154297e-08, + "selector_regret": 5.155801773071289e-06, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.9999945759773254, + "tau": 0.34408156791985406 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5615544380344981, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6514445232612912, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.18171554803848267, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6191661897193406, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 3.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.8182844519615173, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "608016290/train0": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.6553002828082154, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "608016290/train1": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.5900770630713733, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 1.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 1.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 1.274298369884491, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "608016290/train2": { + "base_success": 0.0, + "base_utility": 0.725701630115509, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.3062491108599271, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.6825780272483826, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.725701630115509, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.3174219727516174, + "tau": 0.34408156791985406 + }, + "626486813/train0": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.12362553612050224, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.043898362666368484, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00509834272088483, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.038800019945483655, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.12241578102111816, + "tau": 0.34408156791985406 + }, + "626486813/train1": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.4708417796239817, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.030555277597159147, + "selected_candidate_index": 5.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.006621529464609921, + "selected_utility_gain_over_base": 0.0015231867437250912, + "selector_regret": 0.023933748132549226, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.13502710359171033, + "tau": 0.34408156791985406 + }, + "626486813/train2": { + "base_success": 0.0, + "base_utility": 0.00509834272088483, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.16558238118886948, + "outcome_ptr": 1.0, + "predicted_margin": 0.039552385944957046, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.03856666944921017, + "selected_candidate_index": 0.5, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.00509834272088483, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.03346832672832534, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.1270157117396593, + "tau": 0.34408156791985406 + }, + "636680137/train0": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 1.0, + "predicted_margin": 0.12068125693439798, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.08826130628585815, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.005126260221004486, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0823030173778534, + "tau": 0.34408156791985406 + }, + "636680137/train1": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.21234411275170456, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.06741118431091309, + "selected_candidate_index": 1.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.10315313935279846, + "tau": 0.34408156791985406 + }, + "636680137/train2": { + "base_success": 0.0, + "base_utility": 0.08313504606485367, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.17056432366371155, + "outcome_ptr": 0.0, + "predicted_margin": 0.13211103402371466, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.0756516233086586, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.08313504606485367, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.09491270035505295, + "tau": 0.34408156791985406 + }, + "655884605/train0": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.21643525031550226, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13732868432998657, + "selected_candidate_index": 7.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8626713156700134, + "tau": 0.34408156791985406 + }, + "655884605/train1": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.2994868754670774, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.1651531159877777, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0021464377641677856, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8348468840122223, + "tau": 0.34408156791985406 + }, + "655884605/train2": { + "base_success": 0.0, + "base_utility": 0.16300667822360992, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 1.0, + "predicted_margin": 0.24482913949185411, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.21447330713272095, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.16300667822360992, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.05146662890911102, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.785526692867279, + "tau": 0.34408156791985406 + }, + "672328262/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4623294557049326, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "672328262/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.32509114906662, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "672328262/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.4398883507596493, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 5.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "701667486/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6093046734206291, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 0.0, + "selected_success_gain_over_base": -1.0, + "selected_utility": 0.18805170059204102, + "selected_utility_gain_over_base": -1.811948299407959, + "selector_regret": 1.811948299407959, + "success_selector_gap": 1.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "701667486/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.5422427512573754, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 2.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "701667486/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.6258136711557138, + "proposal_oracle_success": 1.0, + "proposal_oracle_utility": 2.0, + "selected_candidate_index": 4.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.0, + "tau": 0.34408156791985406 + }, + "747260437/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.13550930803858485, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.10839563608169556, + "selected_candidate_index": 6.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.17026761174201965, + "tau": 0.34408156791985406 + }, + "747260437/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.0318722933242058, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.05267423391342163, + "selected_candidate_index": 0.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.22598901391029358, + "tau": 0.34408156791985406 + }, + "747260437/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 0.0, + "hidden_chart_oracle_utility": 0.2786632478237152, + "outcome_ptr": 0.0, + "predicted_margin": 0.04297570610377669, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.12510187923908234, + "selected_candidate_index": 7.0, + "selected_success": 1.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 2.0, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 0.0, + "support_gap": 0.15356136858463287, + "tau": 0.34408156791985406 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.3191869353543992, + "proposal_oracle_success": 0.3333333333333333, + "proposal_oracle_utility": 0.7526603279014429, + "selected_candidate_index": 3.6666666666666665, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.3333333333333333, + "selected_utility": 0.7452354828516642, + "selected_utility_gain_over_base": 0.6276423335075378, + "selector_regret": 0.024625877539316814, + "success_selector_gap": 0.0, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.6553361676633358, + "tau": 0.34408156791985406 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.6666666666666666, + "execute_generated": 0.6666666666666666, + "fallback_rate": 0.3333333333333333, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 1.0, + "predicted_margin": 0.39915235063549687, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.3705032741030057, + "selected_candidate_index": 5.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.13002302000919977, + "selected_utility_gain_over_base": 0.012429870665073395, + "selector_regret": 1.240480254093806, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.03749322146177292, + "tau": 0.34408156791985406 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11759314934412639, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 0.6666666666666666, + "hidden_chart_oracle_utility": 1.4079964955647786, + "outcome_ptr": 0.6666666666666666, + "predicted_margin": 0.2604958289656638, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.349846323331197, + "selected_candidate_index": 4.333333333333333, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.10690136750539143, + "selected_utility_gain_over_base": -0.010691781838734945, + "selector_regret": 1.2574311296145122, + "success_selector_gap": 0.6666666666666666, + "success_support_gap": 0.0, + "support_gap": 0.05815017223358154, + "tau": 0.34408156791985406 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.4445396242683897, + "proposal_oracle_success": 0.5, + "proposal_oracle_utility": 1.0424291878007352, + "selected_candidate_index": 1.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5375678082928061, + "selected_utility_gain_over_base": -0.035631533712148666, + "selector_regret": 0.5135916564613581, + "success_selector_gap": 0.25, + "success_support_gap": 0.0, + "support_gap": 0.06562284985557199, + "tau": 0.34408156791985406 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.25, + "predicted_margin": 0.3626172469935749, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.5951075805351138, + "selected_candidate_index": 2.0, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5778386716265231, + "selected_utility_gain_over_base": 0.004639329621568322, + "selector_regret": 0.025535901775583625, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.5129444571211934, + "tau": 0.34408156791985406 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5731993420049548, + "coverage": 0.5, + "execute_generated": 0.5, + "fallback_rate": 0.5, + "hidden_chart_oracle_success": 0.25, + "hidden_chart_oracle_utility": 0.6737305596470833, + "outcome_ptr": 0.5, + "predicted_margin": 0.33960353103820384, + "proposal_oracle_success": 0.25, + "proposal_oracle_utility": 0.6134321661666036, + "selected_candidate_index": 4.75, + "selected_success": 0.25, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5891513926908374, + "selected_utility_gain_over_base": 0.01595205068588257, + "selector_regret": 0.024280773475766182, + "success_selector_gap": 0.0, + "success_support_gap": 0.25, + "support_gap": 0.49461987148970366, + "tau": 0.34408156791985406 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics_by_task.json b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..178ca04791265259fd051389ea174a85067c1491 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.2, + "base_utility": 0.9498705387115478, + "coverage": 0.4666666666666667, + "execute_generated": 0.4666666666666667, + "fallback_rate": 0.5333333333333333, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.4666666666666667, + "predicted_margin": 0.34879604119920893, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 1.2244940986235937, + "selected_candidate_index": 1.7333333333333334, + "selected_success": 0.26666666666666666, + "selected_success_gain_over_base": 0.06666666666666667, + "selected_utility": 1.0197147250175476, + "selected_utility_gain_over_base": 0.06984418630599976, + "selector_regret": 0.27158633470535276, + "success_selector_gap": 0.2, + "success_support_gap": 0.5333333333333333, + "support_gap": 0.7755059013764064, + "tau": 0.34408156791985406 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.5839591026306152, + "coverage": 0.0, + "execute_generated": 0.0, + "fallback_rate": 1.0, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.0, + "predicted_margin": 0.03600324406049464, + "proposal_oracle_success": 0.0, + "proposal_oracle_utility": 0.13425676027933756, + "selected_candidate_index": 3.0, + "selected_success": 0.0, + "selected_success_gain_over_base": 0.0, + "selected_utility": 0.5839591026306152, + "selected_utility_gain_over_base": 0.0, + "selector_regret": 0.0, + "success_selector_gap": 0.0, + "success_support_gap": 1.0, + "support_gap": 1.8657432397206624, + "tau": 0.34408156791985406 + }, + "PullCube-v1": { + "base_success": 0.36, + "base_utility": 0.7751704612234608, + "coverage": 0.41333333333333333, + "execute_generated": 0.41333333333333333, + "fallback_rate": 0.5866666666666667, + "hidden_chart_oracle_success": 0.52, + "hidden_chart_oracle_utility": 1.1467878860235214, + "outcome_ptr": 0.44, + "predicted_margin": 0.33772887669024154, + "proposal_oracle_success": 0.4666666666666667, + "proposal_oracle_utility": 0.9816135169876119, + "selected_candidate_index": 3.453333333333333, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": -0.02666666666666667, + "selected_utility": 0.7196805840544402, + "selected_utility_gain_over_base": -0.05548987716902047, + "selector_regret": 0.37178792303428054, + "success_selector_gap": 0.18666666666666668, + "success_support_gap": 0.09333333333333334, + "support_gap": 0.23527305972451965, + "tau": 0.3440815679198542 + }, + "PushCube-v1": { + "base_success": 0.16666666666666666, + "base_utility": 0.3924041787783305, + "coverage": 1.0, + "execute_generated": 1.0, + "fallback_rate": 0.0, + "hidden_chart_oracle_success": 0.8333333333333334, + "hidden_chart_oracle_utility": 1.7015800600250561, + "outcome_ptr": 0.7777777777777778, + "predicted_margin": 0.7573891615571136, + "proposal_oracle_success": 0.6666666666666666, + "proposal_oracle_utility": 1.365773618221283, + "selected_candidate_index": 2.611111111111111, + "selected_success": 0.3333333333333333, + "selected_success_gain_over_base": 0.16666666666666666, + "selected_utility": 0.7119888936479887, + "selected_utility_gain_over_base": 0.31958471486965817, + "selector_regret": 0.6537847245732943, + "success_selector_gap": 0.3333333333333333, + "success_support_gap": 0.3333333333333333, + "support_gap": 0.634226381778717, + "tau": 0.3440815679198541 + }, + "StackCube-v1": { + "base_success": 0.2727272727272727, + "base_utility": 0.7763947045261209, + "coverage": 0.3333333333333333, + "execute_generated": 0.3333333333333333, + "fallback_rate": 0.6666666666666666, + "hidden_chart_oracle_success": 1.0, + "hidden_chart_oracle_utility": 2.0, + "outcome_ptr": 0.5757575757575758, + "predicted_margin": 0.31486228576542635, + "proposal_oracle_success": 0.45454545454545453, + "proposal_oracle_utility": 1.0546941955884297, + "selected_candidate_index": 3.8484848484848486, + "selected_success": 0.24242424242424243, + "selected_success_gain_over_base": -0.030303030303030304, + "selected_utility": 0.7367336921619646, + "selected_utility_gain_over_base": -0.03966101236415632, + "selector_regret": 0.3222826457384861, + "success_selector_gap": 0.21212121212121213, + "success_support_gap": 0.5454545454545454, + "support_gap": 0.9453058044115702, + "tau": 0.34408156791985417 + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/report.md b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/report.md new file mode 100644 index 0000000000000000000000000000000000000000..6436c0965b56da24451115d27f08f8fdae4e30d3 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/report.md @@ -0,0 +1,21 @@ +# Nonlinear Train-Calibrated CTT Selector + +Calibration rows: `432` +Fit rows: `277` +Selection rows: `155` +Eval rows: `144` +Selected model: `hgb_classifier` +Selected params: `{"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0}` +Tau: `0.344082` +Feature set: `context_tangent` +Target: `success` + +The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting. + +| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| fit | 0.4729 | 0.5271 | 0.2780 | 0.5126 | 0.4910 | 0.4838 | 0.2058 | 0.0433 | +| selection | 0.4710 | 0.5290 | 0.2774 | 0.3742 | 0.4387 | 0.5032 | 0.2774 | 0.1226 | +| eval | 0.4653 | 0.5347 | 0.2917 | 0.2986 | 0.4792 | 0.5069 | 0.2917 | 0.2083 | + +This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout. diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/split_hash.txt b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..865ad41e869c1bfbc57e4db65ed79e91e6d93081 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/split_hash.txt @@ -0,0 +1,14 @@ +{ + "calibration_target_index": { + "content_hash": "67f7b4a692f7c7e71da24378dc71e3399c5d35afc14c494869eba6087b765c42", + "retrieval_index_allowed": true, + "split": "train", + "split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372" + }, + "eval_target_index": { + "content_hash": "0c38d73dc9bc4ab756e12006671b5d12f1a23358b281e54b3f6a9ae3cbf6e9af", + "retrieval_index_allowed": false, + "split": "test", + "split_hash": "84151df9e88c219b76751677558cc5bbbc19082bf358cea7330158e9e403e25f" + } +} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/table.tex b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..a892790ca06cffe48df542208e13099cf81b5276 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/table.tex @@ -0,0 +1,8 @@ +% Auto-generated by scripts/eval_nonlinear_dominance_selector.py +\begin{tabular}{lrrrrrrrr} +\toprule +Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\ +\midrule +144 & 0.4653 & 0.5347 & 0.2917 & 0.2986 & 0.4792 & 0.5069 & 0.2917 & 0.2083 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/train.log b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/train.log new file mode 100644 index 0000000000000000000000000000000000000000..91c6aedd1cbec32c4734666224762c5793f71b88 --- /dev/null +++ b/workspace/runs/ctt_base_context_obs_nonlinear_dominance_context_tangent_success_envclip_train_to_test/train.log @@ -0,0 +1,5 @@ +trained nonlinear dominance selector on calibration-fit rows only +selected_model_type=hgb_classifier +selected_model_params={"l2_regularization": 0.01, "learning_rate": 0.05, "max_iter": 80, "max_leaf_nodes": 15, "random_state": 0} +tau=0.344082 +calibration_input=runs/ctt_base_context_obs_train_cal_envclip_rollout_comparison/combined_measured_candidates.json