diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/report.md b/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/report.md index 3f48b331a9298ad3c76b59bcc45ec2e819c17997..c815a56ca02ab82390e277a643303660e475ce84 100644 --- a/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/report.md +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/report.md @@ -4,8 +4,24 @@ K: `8` | Metric | N | Micro mean | 95% CI | Task macro | Seed macro | | --- | ---: | ---: | ---: | ---: | ---: | -| branch_car_at_8 | 16 | 0.2645 | [0.0178, 0.6268] | 0.2315 | 0.3012 | +| base_success | 16 | 0.3125 | [0.1250, 0.5625] | 0.2222 | 0.3571 | +| base_utility | 16 | 0.6871 | [0.3147, 1.1643] | 0.5332 | 0.7765 | +| branch_car_at_8 | 16 | 0.2645 | [0.0149, 0.6270] | 0.2315 | 0.3012 | +| hidden_chart_oracle_success_at_8 | 16 | 0.6250 | [0.3750, 0.8125] | 0.7778 | 0.7143 | +| hidden_chart_oracle_utility_at_8 | 16 | 1.3220 | [0.8644, 1.6665] | 1.5982 | 1.4848 | | outcome_ptr_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.2963 | 0.3929 | -| pairwise_causal_calibration_ece | 16 | 0.1330 | [0.0823, 0.1798] | 0.1366 | 0.1312 | -| selector_regret_at_8 | 16 | 0.2645 | [0.0178, 0.6268] | 0.2315 | 0.3012 | -| support_gap_at_8 | 16 | 0.5139 | [0.1567, 0.8696] | 0.9286 | 0.5714 | +| pairwise_causal_calibration_ece | 16 | 0.1330 | [0.0830, 0.1813] | 0.1366 | 0.1312 | +| proposal_oracle_success_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.2963 | 0.4286 | +| proposal_oracle_success_gain_over_base_at_8 | 16 | 0.0625 | [-0.1250, 0.2500] | 0.0741 | 0.0714 | +| proposal_oracle_utility_at_8 | 16 | 0.8081 | [0.3368, 1.2765] | 0.6696 | 0.9134 | +| proposal_oracle_utility_gain_over_base_at_8 | 16 | 0.1210 | [-0.2626, 0.4792] | 0.1364 | 0.1369 | +| selected_success_at_8 | 16 | 0.2500 | [0.0625, 0.5000] | 0.1852 | 0.2857 | +| selected_success_gain_over_base_at_8 | 16 | -0.0625 | [-0.3125, 0.1250] | -0.0370 | -0.0714 | +| selected_utility_at_8 | 16 | 0.5436 | [0.1771, 1.0157] | 0.4381 | 0.6122 | +| selected_utility_gain_over_base_at_8 | 16 | -0.1435 | [-0.6171, 0.2281] | -0.0950 | -0.1643 | +| selector_regret_at_8 | 16 | 0.2645 | [0.0149, 0.6270] | 0.2315 | 0.3012 | +| success_selector_gap_at_8 | 16 | 0.1250 | [0.0000, 0.3125] | 0.1111 | 0.1429 | +| success_support_gap_at_8 | 16 | 0.2500 | [0.0625, 0.5000] | 0.4815 | 0.2857 | +| success_total_car_to_hidden_at_8 | 16 | 0.3750 | [0.1250, 0.6250] | 0.5926 | 0.4286 | +| support_gap_at_8 | 16 | 0.5139 | [0.1583, 0.9626] | 0.9286 | 0.5714 | +| total_car_to_hidden_at_8 | 16 | 0.7784 | [0.3386, 1.2434] | 1.1601 | 0.8726 | diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/table.tex b/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/table.tex index 8a0d2244db3940400c945c004e9b40a703acc008..5f0fa1c3be4243560d13274db684fc7ab5d10a5e 100644 --- a/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/table.tex +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/measured_metrics/table.tex @@ -3,10 +3,26 @@ \toprule Metric & N & Micro mean & CI low & CI high \\ \midrule -branch\_car\_at\_8 & 16 & 0.2645 & 0.0178 & 0.6268 \\ +base\_success & 16 & 0.3125 & 0.1250 & 0.5625 \\ +base\_utility & 16 & 0.6871 & 0.3147 & 1.1643 \\ +branch\_car\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\ +hidden\_chart\_oracle\_success\_at\_8 & 16 & 0.6250 & 0.3750 & 0.8125 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 16 & 1.3220 & 0.8644 & 1.6665 \\ outcome\_ptr\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\ -pairwise\_causal\_calibration\_ece & 16 & 0.1330 & 0.0823 & 0.1798 \\ -selector\_regret\_at\_8 & 16 & 0.2645 & 0.0178 & 0.6268 \\ -support\_gap\_at\_8 & 16 & 0.5139 & 0.1567 & 0.8696 \\ +pairwise\_causal\_calibration\_ece & 16 & 0.1330 & 0.0830 & 0.1813 \\ +proposal\_oracle\_success\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 16 & 0.0625 & -0.1250 & 0.2500 \\ +proposal\_oracle\_utility\_at\_8 & 16 & 0.8081 & 0.3368 & 1.2765 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 16 & 0.1210 & -0.2626 & 0.4792 \\ +selected\_success\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\ +selected\_success\_gain\_over\_base\_at\_8 & 16 & -0.0625 & -0.3125 & 0.1250 \\ +selected\_utility\_at\_8 & 16 & 0.5436 & 0.1771 & 1.0157 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 16 & -0.1435 & -0.6171 & 0.2281 \\ +selector\_regret\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\ +success\_selector\_gap\_at\_8 & 16 & 0.1250 & 0.0000 & 0.3125 \\ +success\_support\_gap\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\ +success\_total\_car\_to\_hidden\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\ +support\_gap\_at\_8 & 16 & 0.5139 & 0.1583 & 0.9626 \\ +total\_car\_to\_hidden\_at\_8 & 16 & 0.7784 & 0.3386 & 1.2434 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/metrics.json b/workspace/runs/ctt_residual_rollout_val16_seed0/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3771e78ba70db6d4e382fa65ad2494b7a3f57ce5 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/metrics.json @@ -0,0 +1,1183 @@ +{ + "checkpoint": "runs/ctt_residual_full_seed0/model.pt", + "k": 8, + "measured_metric_summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6428571428571429, + "low": 0.14285714285714285, + "mean": 0.35714285714285715, + "n": 14, + "num_samples": 500 + }, + "mean": 0.35714285714285715, + "per_seed": { + "1189392917": 0.0, + "1271718007": 1.0, + "1273247072": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1549324805": 1.0, + "1881223807": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2141802816": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "767747424": 0.0, + "77147264": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3333333333333333, + "low": 0.0, + "mean": 0.2222222222222222, + "n": 3, + "num_samples": 500 + }, + "mean": 0.2222222222222222, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 0.3333333333333333 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.5625, + "low": 0.125, + "mean": 0.3125, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.3199879928891147, + "low": 0.34861662624669926, + "mean": 0.7764950924153838, + "n": 14, + "num_samples": 500 + }, + "mean": 0.7764950924153838, + "per_seed": { + "1189392917": 0.10841864347457886, + "1271718007": 2.0, + "1273247072": 0.15342965722084045, + "1375492786": 2.0, + "141984615": 0.050487834960222244, + "1549324805": 2.0, + "1881223807": 0.0320269912481308, + "1997090716": 0.13927479088306427, + "2015956957": 0.058304885402321815, + "2141802816": 0.07080724835395813, + "565369392": 0.19400137662887573, + "565621826": 2.0, + "767747424": 0.06417986564338207, + "77147264": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7250879456599554, + "low": 0.15342965722084045, + "mean": 0.5331895236891729, + "n": 3, + "num_samples": 500 + }, + "mean": 0.5331895236891729, + "per_task": { + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.7210509681867229, + "PushCube-v1": 0.7250879456599554 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 1.1643496139440686, + "low": 0.3147284146398306, + "mean": 0.6870885028038174, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "branch_car_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7187583449163607, + "low": 0.014021761948242784, + "mean": 0.3011761402262242, + "n": 14, + "num_samples": 500 + }, + "mean": 0.3011761402262242, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1375492786": 0.0002854267368093133, + "141984615": 0.09784513339400291, + "1549324805": 0.0, + "1881223807": 0.020236089825630188, + "1997090716": 1.9691698104143143, + "2015956957": 0.003587027546018362, + "2141802816": 0.07543906569480896, + "565369392": 0.0671776756644249, + "565621826": 0.0, + "767747424": 0.012708732858300209, + "77147264": 1.9700170010328293 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6724769944945971, + "low": 0.0, + "mean": 0.231488618779824, + "n": 3, + "num_samples": 500 + }, + "mean": 0.231488618779824, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.02198886184487492, + "PushCube-v1": 0.6724769944945971 + } + }, + "metric": "branch_car_at_8", + "micro": { + "confidence": 0.95, + "high": 0.6269801619346254, + "low": 0.014924341201549396, + "mean": 0.26454760772321606, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9285714285714286, + "low": 0.5, + "mean": 0.7142857142857143, + "n": 14, + "num_samples": 500 + }, + "mean": 0.7142857142857143, + "per_seed": { + "1189392917": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1549324805": 1.0, + "1881223807": 1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2141802816": 1.0, + "565369392": 1.0, + "565621826": 1.0, + "767747424": 0.0, + "77147264": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.3333333333333333, + "mean": 0.7777777777777777, + "n": 3, + "num_samples": 500 + }, + "mean": 0.7777777777777777, + "per_task": { + "PickCube-v1": 1.0, + "PullCube-v1": 0.3333333333333333, + "PushCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.8125, + "low": 0.375, + "mean": 0.625, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8699864170381002, + "low": 1.0907963934753622, + "mean": 1.4848211647144385, + "n": 14, + "num_samples": 500 + }, + "mean": 1.4848211647144385, + "per_seed": { + "1189392917": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.1798098385334015, + "1549324805": 2.0, + "1881223807": 2.0, + "1997090716": 2.0, + "2015956957": 0.16121773421764374, + "2141802816": 2.0, + "565369392": 2.0, + "565621826": 2.0, + "767747424": 0.20301776379346848, + "77147264": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 0.7946368671125836, + "mean": 1.5982122890375277, + "n": 3, + "num_samples": 500 + }, + "mean": 1.5982122890375277, + "per_task": { + "PickCube-v1": 2.0, + "PullCube-v1": 0.7946368671125836, + "PushCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.666475722566247, + "low": 0.8644023519009352, + "mean": 1.3219832377508283, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "outcome_ptr_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6428571428571429, + "low": 0.14285714285714285, + "mean": 0.39285714285714285, + "n": 14, + "num_samples": 500 + }, + "mean": 0.39285714285714285, + "per_seed": { + "1189392917": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1375492786": 0.0, + "141984615": 1.0, + "1549324805": 0.0, + "1881223807": 1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2141802816": 1.0, + "565369392": 0.0, + "565621826": 0.0, + "767747424": 0.5, + "77147264": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6666666666666666, + "low": 0.0, + "mean": 0.2962962962962963, + "n": 3, + "num_samples": 500 + }, + "mean": 0.2962962962962963, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.2222222222222222, + "PushCube-v1": 0.6666666666666666 + } + }, + "metric": "outcome_ptr_at_8", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.125, + "mean": 0.375, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.19134660629025252, + "low": 0.07903649816149518, + "mean": 0.13118677987364852, + "n": 14, + "num_samples": 500 + }, + "mean": 0.13118677987364852, + "per_seed": { + "1189392917": 0.1628408162009477, + "1271718007": 0.0, + "1273247072": 0.15040783499409596, + "1375492786": 0.19928218350064877, + "141984615": 0.0825986241056758, + "1549324805": 0.2212803648424329, + "1881223807": 0.019060791998604134, + "1997090716": 0.4044963838715546, + "2015956957": 0.18582613278258653, + "2141802816": 0.05242798758366053, + "565369392": 0.1760029309478162, + "565621826": 0.0, + "767747424": 0.10628866977652043, + "77147264": 0.07610219762653536 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.15040783499409596, + "low": 0.11915469621355039, + "mean": 0.13664653664383722, + "n": 3, + "num_samples": 500 + }, + "mean": 0.13664653664383722, + "per_task": { + "PickCube-v1": 0.15040783499409596, + "PullCube-v1": 0.1403770787238653, + "PushCube-v1": 0.11915469621355039 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.18134943609804377, + "low": 0.08303117149093137, + "mean": 0.13304560754938663, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "proposal_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7142857142857143, + "low": 0.21428571428571427, + "mean": 0.42857142857142855, + "n": 14, + "num_samples": 500 + }, + "mean": 0.42857142857142855, + "per_seed": { + "1189392917": 1.0, + "1271718007": 1.0, + "1273247072": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1549324805": 1.0, + "1881223807": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2141802816": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "767747424": 0.0, + "77147264": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6666666666666666, + "low": 0.0, + "mean": 0.2962962962962963, + "n": 3, + "num_samples": 500 + }, + "mean": 0.2962962962962963, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.2222222222222222, + "PushCube-v1": 0.6666666666666666 + } + }, + "metric": "proposal_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.125, + "mean": 0.375, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "proposal_oracle_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2857142857142857, + "low": -0.21428571428571427, + "mean": 0.07142857142857142, + "n": 14, + "num_samples": 500 + }, + "mean": 0.07142857142857142, + "per_seed": { + "1189392917": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1549324805": 0.0, + "1881223807": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2141802816": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "767747424": 0.0, + "77147264": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3333333333333333, + "low": -0.1111111111111111, + "mean": 0.07407407407407407, + "n": 3, + "num_samples": 500 + }, + "mean": 0.07407407407407407, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": -0.1111111111111111, + "PushCube-v1": 0.3333333333333333 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.25, + "low": -0.125, + "mean": 0.0625, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.456283768015315, + "low": 0.4903542116434047, + "mean": 0.9134039957092942, + "n": 14, + "num_samples": 500 + }, + "mean": 0.9134039957092942, + "per_seed": { + "1189392917": 2.0, + "1271718007": 2.0, + "1273247072": 0.13278456032276154, + "1375492786": 0.0016440550098195672, + "141984615": 0.1541295200586319, + "1549324805": 2.0, + "1881223807": 0.0513521283864975, + "1997090716": 2.0, + "2015956957": 0.028452674858272076, + "2141802816": 0.12472955882549286, + "565369392": 0.18084704875946045, + "565621826": 2.0, + "767747424": 0.11371639370918274, + "77147264": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3626802812019985, + "low": 0.13278456032276154, + "mean": 0.6696349012846542, + "n": 3, + "num_samples": 500 + }, + "mean": 0.6696349012846542, + "per_task": { + "PickCube-v1": 0.13278456032276154, + "PullCube-v1": 0.5134398623292024, + "PushCube-v1": 1.3626802812019985 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.2764708213871927, + "low": 0.3368462507642107, + "mean": 0.8081140630310983, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5669070603325963, + "low": -0.4169755119780478, + "mean": 0.1369089032939103, + "n": 14, + "num_samples": 500 + }, + "mean": 0.1369089032939103, + "per_seed": { + "1189392917": 1.8915813565254211, + "1271718007": 0.0, + "1273247072": -0.02064509689807892, + "1375492786": -1.9983559449901804, + "141984615": 0.10364168509840965, + "1549324805": 0.0, + "1881223807": 0.0193251371383667, + "1997090716": 1.8607252091169357, + "2015956957": -0.02985221054404974, + "2141802816": 0.05392231047153473, + "565369392": -0.013154327869415283, + "565621826": 0.0, + "767747424": 0.04953652806580067, + "77147264": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6375923355420431, + "low": -0.20761110585752043, + "mean": 0.13644537759548123, + "n": 3, + "num_samples": 500 + }, + "mean": 0.13644537759548123, + "per_task": { + "PickCube-v1": -0.02064509689807892, + "PullCube-v1": -0.20761110585752046, + "PushCube-v1": 0.6375923355420431 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.47923582210205495, + "low": -0.26257677190733375, + "mean": 0.12102556022728095, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5, + "low": 0.07142857142857142, + "mean": 0.2857142857142857, + "n": 14, + "num_samples": 500 + }, + "mean": 0.2857142857142857, + "per_seed": { + "1189392917": 1.0, + "1271718007": 1.0, + "1273247072": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1549324805": 1.0, + "1881223807": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2141802816": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "767747424": 0.0, + "77147264": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3333333333333333, + "low": 0.0, + "mean": 0.1851851851851852, + "n": 3, + "num_samples": 500 + }, + "mean": 0.1851851851851852, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.2222222222222222, + "PushCube-v1": 0.3333333333333333 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5, + "low": 0.0625, + "mean": 0.25, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.14285714285714285, + "low": -0.2857142857142857, + "mean": -0.07142857142857142, + "n": 14, + "num_samples": 500 + }, + "mean": -0.07142857142857142, + "per_seed": { + "1189392917": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1549324805": 0.0, + "1881223807": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2141802816": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "767747424": 0.0, + "77147264": -1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.0, + "low": -0.1111111111111111, + "mean": -0.037037037037037035, + "n": 3, + "num_samples": 500 + }, + "mean": -0.037037037037037035, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": -0.1111111111111111, + "PushCube-v1": 0.0 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.125, + "low": -0.3125, + "mean": -0.0625, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0413506987305092, + "low": 0.18699326519189136, + "mean": 0.61222785548307, + "n": 14, + "num_samples": 500 + }, + "mean": 0.61222785548307, + "per_seed": { + "1189392917": 2.0, + "1271718007": 2.0, + "1273247072": 0.13278456032276154, + "1375492786": 0.001358628273010254, + "141984615": 0.05628438666462898, + "1549324805": 2.0, + "1881223807": 0.03111603856086731, + "1997090716": 0.03083018958568573, + "2015956957": 0.024865647312253714, + "2141802816": 0.0492904931306839, + "565369392": 0.11366937309503555, + "565621826": 2.0, + "767747424": 0.10100766085088253, + "77147264": 0.029982998967170715 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6902032867074013, + "low": 0.13278456032276154, + "mean": 0.4381462825048301, + "n": 3, + "num_samples": 500 + }, + "mean": 0.4381462825048301, + "per_task": { + "PickCube-v1": 0.13278456032276154, + "PullCube-v1": 0.4914510004843275, + "PushCube-v1": 0.6902032867074013 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.015705294383224, + "low": 0.17710547981550917, + "mean": 0.5435664553078823, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2521127736129399, + "low": -0.5916834228805133, + "mean": -0.16426723693231388, + "n": 14, + "num_samples": 500 + }, + "mean": -0.16426723693231388, + "per_seed": { + "1189392917": 1.8915813565254211, + "1271718007": 0.0, + "1273247072": -0.02064509689807892, + "1375492786": -1.9986413717269897, + "141984615": 0.005796551704406738, + "1549324805": 0.0, + "1881223807": -0.0009109526872634888, + "1997090716": -0.10844460129737854, + "2015956957": -0.0334392380900681, + "2141802816": -0.02151675522327423, + "565369392": -0.08033200353384018, + "565621826": 0.0, + "767747424": 0.03682779520750046, + "77147264": -1.9700170010328293 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.02064509689807892, + "low": -0.22959996770239535, + "mean": -0.09504324118434278, + "n": 3, + "num_samples": 500 + }, + "mean": -0.09504324118434278, + "per_task": { + "PickCube-v1": -0.02064509689807892, + "PullCube-v1": -0.22959996770239538, + "PushCube-v1": -0.03488465895255407 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.22805603814776987, + "low": -0.617137425870169, + "mean": -0.1435220474959351, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7187583449163607, + "low": 0.014021761948242784, + "mean": 0.3011761402262242, + "n": 14, + "num_samples": 500 + }, + "mean": 0.3011761402262242, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1375492786": 0.0002854267368093133, + "141984615": 0.09784513339400291, + "1549324805": 0.0, + "1881223807": 0.020236089825630188, + "1997090716": 1.9691698104143143, + "2015956957": 0.003587027546018362, + "2141802816": 0.07543906569480896, + "565369392": 0.0671776756644249, + "565621826": 0.0, + "767747424": 0.012708732858300209, + "77147264": 1.9700170010328293 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6724769944945971, + "low": 0.0, + "mean": 0.231488618779824, + "n": 3, + "num_samples": 500 + }, + "mean": 0.231488618779824, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.02198886184487492, + "PushCube-v1": 0.6724769944945971 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.6269801619346254, + "low": 0.014924341201549396, + "mean": 0.26454760772321606, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.35714285714285715, + "low": 0.0, + "mean": 0.14285714285714285, + "n": 14, + "num_samples": 500 + }, + "mean": 0.14285714285714285, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1549324805": 0.0, + "1881223807": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2141802816": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "767747424": 0.0, + "77147264": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3333333333333333, + "low": 0.0, + "mean": 0.1111111111111111, + "n": 3, + "num_samples": 500 + }, + "mean": 0.1111111111111111, + "per_task": { + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.3333333333333333 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.3125, + "low": 0.0, + "mean": 0.125, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5, + "low": 0.07142857142857142, + "mean": 0.2857142857142857, + "n": 14, + "num_samples": 500 + }, + "mean": 0.2857142857142857, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1549324805": 0.0, + "1881223807": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2141802816": 1.0, + "565369392": 1.0, + "565621826": 0.0, + "767747424": 0.0, + "77147264": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.1111111111111111, + "mean": 0.48148148148148145, + "n": 3, + "num_samples": 500 + }, + "mean": 0.48148148148148145, + "per_task": { + "PickCube-v1": 1.0, + "PullCube-v1": 0.1111111111111111, + "PushCube-v1": 0.3333333333333333 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5, + "low": 0.0625, + "mean": 0.25, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7142857142857143, + "low": 0.14285714285714285, + "mean": 0.42857142857142855, + "n": 14, + "num_samples": 500 + }, + "mean": 0.42857142857142855, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1549324805": 0.0, + "1881223807": 1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2141802816": 1.0, + "565369392": 1.0, + "565621826": 0.0, + "767747424": 0.0, + "77147264": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.1111111111111111, + "mean": 0.5925925925925926, + "n": 3, + "num_samples": 500 + }, + "mean": 0.5925925925925926, + "per_task": { + "PickCube-v1": 1.0, + "PullCube-v1": 0.1111111111111111, + "PushCube-v1": 0.6666666666666666 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.625, + "low": 0.125, + "mean": 0.375, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.00120393927708, + "low": 0.16012767239174405, + "mean": 0.5714171690051444, + "n": 14, + "num_samples": 500 + }, + "mean": 0.5714171690051444, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 1.8672154396772385, + "1375492786": 0.24180691444780678, + "141984615": 0.025680318474769592, + "1549324805": 0.0, + "1881223807": 1.9486478716135025, + "1997090716": 0.0, + "2015956957": 0.13276505935937166, + "2141802816": 1.8752704411745071, + "565369392": 1.8191529512405396, + "565621826": 0.0, + "767747424": 0.08930137008428574, + "77147264": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.8672154396772385, + "low": 0.28119700478338117, + "mean": 0.9285773877528739, + "n": 3, + "num_samples": 500 + }, + "mean": 0.9285773877528739, + "per_task": { + "PickCube-v1": 1.8672154396772385, + "PullCube-v1": 0.28119700478338117, + "PushCube-v1": 0.6373197187980016 + } + }, + "metric": "support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9626226120162755, + "low": 0.1582781979450374, + "mean": 0.5138691747197299, + "n": 16, + "num_samples": 500 + }, + "n": 16 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4018085617904685, + "low": 0.3681092694667833, + "mean": 0.8725933092313686, + "n": 14, + "num_samples": 500 + }, + "mean": 0.8725933092313686, + "per_seed": { + "1189392917": 0.0, + "1271718007": 0.0, + "1273247072": 1.8672154396772385, + "1375492786": 0.2420923411846161, + "141984615": 0.1235254518687725, + "1549324805": 0.0, + "1881223807": 1.9688839614391327, + "1997090716": 1.9691698104143143, + "2015956957": 0.13635208690539002, + "2141802816": 1.950709506869316, + "565369392": 1.8863306269049644, + "565621826": 0.0, + "767747424": 0.10201010294258595, + "77147264": 1.9700170010328293 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.8672154396772385, + "low": 0.3031858666282561, + "mean": 1.1600660065326978, + "n": 3, + "num_samples": 500 + }, + "mean": 1.1600660065326978, + "per_task": { + "PickCube-v1": 1.8672154396772385, + "PullCube-v1": 0.3031858666282561, + "PushCube-v1": 1.3097967132925987 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 1.2434286201023497, + "low": 0.338613064144738, + "mean": 0.778416782442946, + "n": 16, + "num_samples": 500 + }, + "n": 16 + } + }, + "num_rows": 16, + "report_type": "ctt_generated_measured_rollout", + "schema_version": 1, + "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957", + "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "success_summary": { + "base_success_rate": 0.3125, + "base_utility_mean": 0.6870885028038174, + "hidden_chart_oracle_success_rate": 0.625, + "hidden_chart_oracle_utility_mean": 1.3219832377508283, + "max_restore_error": 1.4901161193847656e-08, + "proposal_oracle_success_gain_over_base": 0.0625, + "proposal_oracle_success_rate": 0.375, + "proposal_oracle_utility_mean": 0.8081140630310983, + "selected_success_gain_over_base": -0.0625, + "selected_success_rate": 0.25, + "selected_utility_mean": 0.5435664553078823, + "success_selector_gap": 0.125, + "success_support_gap": 0.25 + }, + "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42", + "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381" +} diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..098debe3a72ee44fcb276ad2f6ab8592241602d2 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_seed.json @@ -0,0 +1,324 @@ +{ + "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1271718007": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 + }, + "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0002854267368093133, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, + "selector_regret_at_8": 0.0002854267368093133, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24180691444780678, + "total_car_to_hidden_at_8": 0.2420923411846161 + }, + "141984615": { + "base_success": 0.0, + "base_utility": 0.050487834960222244, + "branch_car_at_8": 0.09784513339400291, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0825986241056758, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1541295200586319, + "proposal_oracle_utility_gain_over_base_at_8": 0.10364168509840965, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05628438666462898, + "selected_utility_gain_over_base_at_8": 0.005796551704406738, + "selector_regret_at_8": 0.09784513339400291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.025680318474769592, + "total_car_to_hidden_at_8": 0.1235254518687725 + }, + "1549324805": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2212803648424329, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.020236089825630188, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, + "selector_regret_at_8": 0.020236089825630188, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9486478716135025, + "total_car_to_hidden_at_8": 1.9688839614391327 + }, + "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 1.9691698104143143, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, + "selector_regret_at_8": 1.9691698104143143, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9691698104143143 + }, + "2015956957": { + "base_success": 0.0, + "base_utility": 0.058304885402321815, + "branch_car_at_8": 0.003587027546018362, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.16121773421764374, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.18582613278258653, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.028452674858272076, + "proposal_oracle_utility_gain_over_base_at_8": -0.02985221054404974, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.024865647312253714, + "selected_utility_gain_over_base_at_8": -0.0334392380900681, + "selector_regret_at_8": 0.003587027546018362, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.13276505935937166, + "total_car_to_hidden_at_8": 0.13635208690539002 + }, + "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 0.07543906569480896, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, + "selector_regret_at_8": 0.07543906569480896, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8752704411745071, + "total_car_to_hidden_at_8": 1.950709506869316 + }, + "565369392": { + "base_success": 0.0, + "base_utility": 0.19400137662887573, + "branch_car_at_8": 0.0671776756644249, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.1760029309478162, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.18084704875946045, + "proposal_oracle_utility_gain_over_base_at_8": -0.013154327869415283, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.11366937309503555, + "selected_utility_gain_over_base_at_8": -0.08033200353384018, + "selector_regret_at_8": 0.0671776756644249, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8191529512405396, + "total_car_to_hidden_at_8": 1.8863306269049644 + }, + "565621826": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "767747424": { + "base_success": 0.0, + "base_utility": 0.06417986564338207, + "branch_car_at_8": 0.012708732858300209, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20301776379346848, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.10628866977652043, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11371639370918274, + "proposal_oracle_utility_gain_over_base_at_8": 0.04953652806580067, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10100766085088253, + "selected_utility_gain_over_base_at_8": 0.03682779520750046, + "selector_regret_at_8": 0.012708732858300209, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.08930137008428574, + "total_car_to_hidden_at_8": 0.10201010294258595 + }, + "77147264": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9700170010328293, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07610219762653536, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.029982998967170715, + "selected_utility_gain_over_base_at_8": -1.9700170010328293, + "selector_regret_at_8": 1.9700170010328293, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9700170010328293 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..62c8a4732be866af5115a03c055866608c3f23b6 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/metrics_by_task.json @@ -0,0 +1,71 @@ +{ + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 + }, + "PullCube-v1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7210509681867229, + "branch_car_at_8": 0.02198886184487492, + "hidden_chart_oracle_success_at_8": 0.3333333333333333, + "hidden_chart_oracle_utility_at_8": 0.7946368671125836, + "outcome_ptr_at_8": 0.2222222222222222, + "pairwise_causal_calibration_ece": 0.1403770787238653, + "proposal_oracle_success_at_8": 0.2222222222222222, + "proposal_oracle_success_gain_over_base_at_8": -0.1111111111111111, + "proposal_oracle_utility_at_8": 0.5134398623292024, + "proposal_oracle_utility_gain_over_base_at_8": -0.20761110585752046, + "selected_success_at_8": 0.2222222222222222, + "selected_success_gain_over_base_at_8": -0.1111111111111111, + "selected_utility_at_8": 0.4914510004843275, + "selected_utility_gain_over_base_at_8": -0.22959996770239538, + "selector_regret_at_8": 0.02198886184487492, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.1111111111111111, + "success_total_car_to_hidden_at_8": 0.1111111111111111, + "support_gap_at_8": 0.28119700478338117, + "total_car_to_hidden_at_8": 0.3031858666282561 + }, + "PushCube-v1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7250879456599554, + "branch_car_at_8": 0.6724769944945971, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.11915469621355039, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.3333333333333333, + "proposal_oracle_utility_at_8": 1.3626802812019985, + "proposal_oracle_utility_gain_over_base_at_8": 0.6375923355420431, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6902032867074013, + "selected_utility_gain_over_base_at_8": -0.03488465895255407, + "selector_regret_at_8": 0.6724769944945971, + "success_selector_gap_at_8": 0.3333333333333333, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.6373197187980016, + "total_car_to_hidden_at_8": 1.3097967132925987 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/source_data_hash.txt b/workspace/runs/ctt_residual_rollout_val16_seed0/source_data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6cc02ca1808e0515839939b88beab0acaad5cbc --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/source_data_hash.txt @@ -0,0 +1 @@ +90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957 diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/source_split_hash.txt b/workspace/runs/ctt_residual_rollout_val16_seed0/source_split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..26988e766c6751325c7f72d33b199bd133350cde --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/source_split_hash.txt @@ -0,0 +1 @@ +867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372 diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/split_hash.txt b/workspace/runs/ctt_residual_rollout_val16_seed0/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..8533e80f235606cef4ba0119a3ac3806d63a0983 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/split_hash.txt @@ -0,0 +1 @@ +9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381 diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/table.tex b/workspace/runs/ctt_residual_rollout_val16_seed0/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..5f0fa1c3be4243560d13274db684fc7ab5d10a5e --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/table.tex @@ -0,0 +1,28 @@ +% Auto-generated by scripts/eval_metrics.py +\begin{tabular}{lrrrr} +\toprule +Metric & N & Micro mean & CI low & CI high \\ +\midrule +base\_success & 16 & 0.3125 & 0.1250 & 0.5625 \\ +base\_utility & 16 & 0.6871 & 0.3147 & 1.1643 \\ +branch\_car\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\ +hidden\_chart\_oracle\_success\_at\_8 & 16 & 0.6250 & 0.3750 & 0.8125 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 16 & 1.3220 & 0.8644 & 1.6665 \\ +outcome\_ptr\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\ +pairwise\_causal\_calibration\_ece & 16 & 0.1330 & 0.0830 & 0.1813 \\ +proposal\_oracle\_success\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 16 & 0.0625 & -0.1250 & 0.2500 \\ +proposal\_oracle\_utility\_at\_8 & 16 & 0.8081 & 0.3368 & 1.2765 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 16 & 0.1210 & -0.2626 & 0.4792 \\ +selected\_success\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\ +selected\_success\_gain\_over\_base\_at\_8 & 16 & -0.0625 & -0.3125 & 0.1250 \\ +selected\_utility\_at\_8 & 16 & 0.5436 & 0.1771 & 1.0157 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 16 & -0.1435 & -0.6171 & 0.2281 \\ +selector\_regret\_at\_8 & 16 & 0.2645 & 0.0149 & 0.6270 \\ +success\_selector\_gap\_at\_8 & 16 & 0.1250 & 0.0000 & 0.3125 \\ +success\_support\_gap\_at\_8 & 16 & 0.2500 & 0.0625 & 0.5000 \\ +success\_total\_car\_to\_hidden\_at\_8 & 16 & 0.3750 & 0.1250 & 0.6250 \\ +support\_gap\_at\_8 & 16 & 0.5139 & 0.1583 & 0.9626 \\ +total\_car\_to\_hidden\_at\_8 & 16 & 0.7784 & 0.3386 & 1.2434 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val16_seed0/train.log b/workspace/runs/ctt_residual_rollout_val16_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..921e09ec7e894ba5250fa0a627691cdfb6e25e63 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val16_seed0/train.log @@ -0,0 +1 @@ +CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed0/model.pt diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/data_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed0/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..a201880dd0bef2025b061d137204bb9db2f137c2 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/data_hash.txt @@ -0,0 +1 @@ +fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/eval.log b/workspace/runs/ctt_residual_rollout_val69_seed0/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..a3cf59a5e9ca1490ba2c9dcc3f6449016ae26739 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/eval.log @@ -0,0 +1,668 @@ +start +importing gymnasium/mani_skill +imported gymnasium/mani_skill +loaded checkpoint=runs/ctt_residual_full_seed0/model.pt +loaded charts source=301 target=69 +generated proposals rows=69 total=552 +env init sequential task=LiftPegUpright-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=LiftPegUpright-v1 +env init sequential task=LiftPegUpright-v1 batch=2 sim=physx_cpu render=cpu +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=LiftPegUpright-v1 +env init sequential task=PickCube-v1 batch=1 sim=physx_cpu render=cpu +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PickCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=2 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=1 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=3 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +rollout complete rows=69 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics.json b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics.json index ae9596fcc202d1f5da3e026bb8200673a6460a2c..e677a7b4dbe1263a2f8591413192cf38e0966e77 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics.json @@ -5,766 +5,2050 @@ "num_rows": 69, "rows": [ { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.4897426515817642, "chart_id": "ms-484ea653bbca2de3a4a2", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18322944379343714, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7187411189079285, + "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766, "seed": "1231780997", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22899846732616425, + "selected_utility_gain_over_base_at_8": -0.498007670044899, "selector_regret_at_8": 0.4897426515817642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2812588810920715, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7710015326738358 }, { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.26188503205776215, "chart_id": "ms-4b4cd12d70c0615e0930", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08963387331564551, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4761262834072113, + "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267, "seed": "1025740212", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21424125134944916, + "selected_utility_gain_over_base_at_8": -0.3141324073076248, "selector_regret_at_8": 0.26188503205776215, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.5238737165927887, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7857587486505508 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-6e268d1e2915740816a1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.09930428958964055, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1475440973", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.7239725589752197, "branch_car_at_8": 0.033253490924835205, "chart_id": "ms-8864796e31bbe93d7c5d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10932818664818449, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7250897884368896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219, "seed": "551504635", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6918362975120544, + "selected_utility_gain_over_base_at_8": -0.03213626146316528, "selector_regret_at_8": 0.033253490924835205, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2749102115631104, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.3081637024879456 }, { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.38470593094825745, "chart_id": "ms-b04b489f4fbad93b65ca", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1616344352768096, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6646121144294739, + "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012, "seed": "1113268940", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.27990618348121643, + "selected_utility_gain_over_base_at_8": -0.14240634441375732, "selector_regret_at_8": 0.38470593094825745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.3353878855705261, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7200938165187836 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c663f7e73e7e8376a05c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.015990866806051107, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1528740417", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.0, "chart_id": "ms-2458fcfffb3f96c6ca1a", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, "seed": "1273247072", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8672154396772385, - "task_id": "PickCube-v1" + "task_id": "PickCube-v1", + "total_car_to_hidden_at_8": 1.8672154396772385 }, { + "base_success": 0.0, + "base_utility": 0.050487834960222244, "branch_car_at_8": 0.09784513339400291, "chart_id": "ms-0345d8ac530f88cefd61", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0825986241056758, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1541295200586319, + "proposal_oracle_utility_gain_over_base_at_8": 0.10364168509840965, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05628438666462898, + "selected_utility_gain_over_base_at_8": 0.005796551704406738, "selector_regret_at_8": 0.09784513339400291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.025680318474769592, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1235254518687725 }, { + "base_success": 0.0, + "base_utility": 0.07743936777114868, "branch_car_at_8": 0.003723899833858013, "chart_id": "ms-0819eb89e627595e7e1b", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.14572584629058838, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2449440706869753, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01781417988240719, + "proposal_oracle_utility_gain_over_base_at_8": -0.05962518788874149, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.014090280048549175, + "selected_utility_gain_over_base_at_8": -0.0633490877225995, "selector_regret_at_8": 0.003723899833858013, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1279116664081812, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1316355662420392 }, { + "base_success": 0.0, + "base_utility": 0.19400137662887573, "branch_car_at_8": 0.0671776756644249, "chart_id": "ms-0dee962a5901fb8b5bbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1760029309478162, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.18084704875946045, + "proposal_oracle_utility_gain_over_base_at_8": -0.013154327869415283, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.11366937309503555, + "selected_utility_gain_over_base_at_8": -0.08033200353384018, "selector_regret_at_8": 0.0671776756644249, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8191529512405396, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8863306269049644 }, { + "base_success": 0.0, + "base_utility": 0.04583283141255379, "branch_car_at_8": 0.0, "chart_id": "ms-126eb8b36769f9ba4671", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1985912322998047, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13211125003864038, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.14532220363616943, + "proposal_oracle_utility_gain_over_base_at_8": 0.09948937222361565, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14532220363616943, + "selected_utility_gain_over_base_at_8": 0.09948937222361565, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.053269028663635254, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.053269028663635254 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-16661b2fd2c85f57a66c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2212803648424329, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002854267368093133, "chart_id": "ms-20e1efbb745fd02abc95", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, "seed": "1375492786", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, "selector_regret_at_8": 0.0002854267368093133, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24180691444780678, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2420923411846161 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-2dc5138251afd26a040d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565621826", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08252689987421036, "branch_car_at_8": 0.025417465716600418, "chart_id": "ms-387e6771a55be98f1a59", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20744429528713226, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08046608951440049, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08211058378219604, + "proposal_oracle_utility_gain_over_base_at_8": -0.00041631609201431274, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05669311806559563, + "selected_utility_gain_over_base_at_8": -0.02583378180861473, "selector_regret_at_8": 0.025417465716600418, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.12533371150493622, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15075117722153664 }, { + "base_success": 0.0, + "base_utility": 0.03917040303349495, "branch_car_at_8": 0.003450155258178711, "chart_id": "ms-38f7298184a2c5005e94", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1767096221446991, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12670819487819776, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03909116983413696, + "proposal_oracle_utility_gain_over_base_at_8": -7.923319935798645e-05, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03564101457595825, + "selected_utility_gain_over_base_at_8": -0.0035293884575366974, "selector_regret_at_8": 0.003450155258178711, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13761845231056213, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.14106860756874084 }, { + "base_success": 0.0, + "base_utility": 0.009124159812927246, "branch_car_at_8": 0.009075979702174664, "chart_id": "ms-482c5c8d208b5907a2c2", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1834145486354828, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.06830739992931822, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.014053802005946636, + "proposal_oracle_utility_gain_over_base_at_8": 0.00492964219301939, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.004977822303771973, + "selected_utility_gain_over_base_at_8": -0.0041463375091552734, "selector_regret_at_8": 0.009075979702174664, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16936074662953615, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.17843672633171082 }, { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.0, "chart_id": "ms-4da03300d92420e80ed0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15825371494561297, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038, "seed": "1747178030", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8536934107542038, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-57be9ae181c077e7821f", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.142949998938739, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.1544450968503952, "branch_car_at_8": 0.0, "chart_id": "ms-6388b5d2356c556c3e55", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09390309844881675, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8455549031496048, "seed": "2015956957", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8455549031496048, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.06173701956868172, "branch_car_at_8": 0.038424551486968994, "chart_id": "ms-638d2ba1917eff4d09e0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.11176212877035141, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.47662120577716466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.054154932498931885, + "proposal_oracle_utility_gain_over_base_at_8": -0.007582087069749832, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01573038101196289, + "selected_utility_gain_over_base_at_8": -0.046006638556718826, "selector_regret_at_8": 0.038424551486968994, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.057607196271419525, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.09603174775838852 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-652ad552ba98d2736620", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2367449700832367, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15432521715768022, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "767747424", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03784912824630737, "branch_car_at_8": 0.04759359359741211, "chart_id": "ms-674fad0e7c042d23629e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.15573063492774963, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.3308470025433796, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11711406707763672, + "proposal_oracle_utility_gain_over_base_at_8": 0.07926493883132935, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06952047348022461, + "selected_utility_gain_over_base_at_8": 0.031671345233917236, "selector_regret_at_8": 0.04759359359741211, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.038616567850112915, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.08621016144752502 }, { + "base_success": 0.0, + "base_utility": 0.12791956961154938, "branch_car_at_8": 0.13747439347207546, "chart_id": "ms-728b865677134d3788e3", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.22658847272396088, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15394090571485494, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1656186729669571, + "proposal_oracle_utility_gain_over_base_at_8": 0.037699103355407715, "seed": "750448296", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02814427949488163, + "selected_utility_gain_over_base_at_8": -0.09977529011666775, "selector_regret_at_8": 0.13747439347207546, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.060969799757003784, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.19844419322907925 }, { + "base_success": 0.0, + "base_utility": 0.20529556274414062, "branch_car_at_8": 0.1261332780122757, "chart_id": "ms-7ea98b8f4d2220347ee0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.27989473938941956, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.25933831347005637, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.17496724426746368, + "proposal_oracle_utility_gain_over_base_at_8": -0.03032831847667694, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04883396625518799, + "selected_utility_gain_over_base_at_8": -0.15646159648895264, "selector_regret_at_8": 0.1261332780122757, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.10492749512195587, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.23106077313423157 }, { + "base_success": 0.0, + "base_utility": 0.18213583528995514, "branch_car_at_8": 0.0009371041087433696, "chart_id": "ms-7f57045c114ddad153b5", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.227622389793396, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12290390946733898, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0028197963256388903, + "proposal_oracle_utility_gain_over_base_at_8": -0.17931603896431625, "seed": "278008524", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0018826922168955207, + "selected_utility_gain_over_base_at_8": -0.18025314307305962, "selector_regret_at_8": 0.0009371041087433696, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2248025934677571, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22573969757650048 }, { + "base_success": 0.0, + "base_utility": 0.0056487321853637695, "branch_car_at_8": 0.004609187599271536, "chart_id": "ms-81f4e8858b2ba90aea2e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1631626933813095, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1605827188396785, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01193171739578247, + "proposal_oracle_utility_gain_over_base_at_8": 0.006282985210418701, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007322529796510935, + "selected_utility_gain_over_base_at_8": 0.0016737976111471653, "selector_regret_at_8": 0.004609187599271536, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.15123097598552704, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15584016358479857 }, { + "base_success": 0.0, + "base_utility": 0.00698717450723052, "branch_car_at_8": 0.06059660017490387, "chart_id": "ms-8a72b73ac452bc190aeb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.10329639911651611, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.25922105796565, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22011399269104004, + "proposal_oracle_utility_gain_over_base_at_8": 0.21312681818380952, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15951739251613617, + "selected_utility_gain_over_base_at_8": 0.15253021800890565, "selector_regret_at_8": 0.06059660017490387, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08285123109817505, "branch_car_at_8": 0.0, "chart_id": "ms-a96bca1ef0b74f686018", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1572066992521286, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1273789448476313, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08222240209579468, + "proposal_oracle_utility_gain_over_base_at_8": -0.0006288290023803711, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08222240209579468, + "selected_utility_gain_over_base_at_8": -0.0006288290023803711, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.07498429715633392, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.07498429715633392 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-ac4fb33f3ab4db853995", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.23392411403598115, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565369392", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c31b98b3d3702c6f9c8c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "2054899126", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.037537217140197754, "branch_car_at_8": 0.0518533973954618, "chart_id": "ms-c832439976a37f52d094", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2594742774963379, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.067807253260605, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.056055743247270584, + "proposal_oracle_utility_gain_over_base_at_8": 0.01851852610707283, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.004202345851808786, + "selected_utility_gain_over_base_at_8": -0.03333487128838897, "selector_regret_at_8": 0.0518533973954618, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2034185342490673, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2552719316445291 }, { + "base_success": 0.0, + "base_utility": 0.09138330072164536, "branch_car_at_8": 0.010477880015969276, "chart_id": "ms-c96b415f083f60857334", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.26426148414611816, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1353708462114277, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01781187392771244, + "proposal_oracle_utility_gain_over_base_at_8": -0.07357142679393291, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007333993911743164, + "selected_utility_gain_over_base_at_8": -0.08404930680990219, "selector_regret_at_8": 0.010477880015969276, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24644961021840572, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.256927490234375 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9274959191679955, "chart_id": "ms-d73fd5fce55fb58ef8e9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.6159370955924921, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "534717141", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07250408083200455, + "selected_utility_gain_over_base_at_8": -1.9274959191679955, "selector_regret_at_8": 1.9274959191679955, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9274959191679955 }, { + "base_success": 0.0, + "base_utility": 0.14809735119342804, "branch_car_at_8": 0.0016585588455200195, "chart_id": "ms-da142025733b53fcf7eb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2643269896507263, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11781174912716066, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0028871893882751465, + "proposal_oracle_utility_gain_over_base_at_8": -0.1452101618051529, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.001228630542755127, + "selected_utility_gain_over_base_at_8": -0.1468687206506729, "selector_regret_at_8": 0.0016585588455200195, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.26143980026245117, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2630983591079712 }, { + "base_success": 0.0, + "base_utility": 0.1362423449754715, "branch_car_at_8": 0.0102931447327137, "chart_id": "ms-e2bad6e99e8a281207cc", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20825156569480896, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.21725827933405525, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.028299549594521523, + "proposal_oracle_utility_gain_over_base_at_8": -0.10794279538094997, "seed": "1549324805", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018006404861807823, + "selected_utility_gain_over_base_at_8": -0.11823594011366367, "selector_regret_at_8": 0.0102931447327137, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.17995201610028744, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.19024516083300114 }, { + "base_success": 0.0, + "base_utility": 0.09695577621459961, "branch_car_at_8": 0.06717542745172977, "chart_id": "ms-e30fd80adc089e9f697a", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18920521438121796, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.10857618383415296, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09409594535827637, + "proposal_oracle_utility_gain_over_base_at_8": -0.002859830856323242, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.026920517906546593, + "selected_utility_gain_over_base_at_8": -0.07003525830805302, "selector_regret_at_8": 0.06717542745172977, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09510926902294159, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.16228469647467136 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-e9ed5d488afdfd6449d1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15838532590498614, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "626486813", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.09514755010604858, "branch_car_at_8": 0.01717013120651245, "chart_id": "ms-ec013396e4aff6170ee1", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24502670764923096, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13327079696269858, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04836767911911011, + "proposal_oracle_utility_gain_over_base_at_8": -0.04677987098693848, "seed": "750448296", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.031197547912597656, + "selected_utility_gain_over_base_at_8": -0.06395000219345093, "selector_regret_at_8": 0.01717013120651245, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.19665902853012085, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2138291597366333 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-ee30ebaf98622759d888", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1699944666253151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012414653785526752, + "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732, "seed": "636680137", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.012414653785526752, + "selected_utility_gain_over_base_at_8": -1.9875853462144732, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9875853462144732, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9875853462144732 }, { + "base_success": 0.0, + "base_utility": 0.026694953441619873, "branch_car_at_8": 0.03375059738755226, "chart_id": "ms-f3f1d07cfd59b823da47", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.17124073207378387, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13684733372604946, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0821961984038353, + "proposal_oracle_utility_gain_over_base_at_8": 0.055501244962215424, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.048445601016283035, + "selected_utility_gain_over_base_at_8": 0.021750647574663162, "selector_regret_at_8": 0.03375059738755226, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.08904453366994858, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.12279513105750084 }, { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.07543906569480896, "chart_id": "ms-081952a63087fc8f83cd", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, "seed": "2141802816", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, "selector_regret_at_8": 0.07543906569480896, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8752704411745071, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.950709506869316 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9700170010328293, "chart_id": "ms-121bc2e6676c1ac10911", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07610219762653536, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "77147264", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.029982998967170715, + "selected_utility_gain_over_base_at_8": -1.9700170010328293, "selector_regret_at_8": 1.9700170010328293, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9700170010328293 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-22f153f0d4fa656018e6", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1271718007", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.020236089825630188, "chart_id": "ms-27c9f70e97b8850cc3ab", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, "seed": "1881223807", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, "selector_regret_at_8": 0.020236089825630188, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9486478716135025, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9688839614391327 }, { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, "chart_id": "ms-2867deaca7e98582620b", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, "seed": "1189392917", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 1.9691698104143143, "chart_id": "ms-365d8c6de6de16f66744", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, "seed": "1997090716", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, "selector_regret_at_8": 1.9691698104143143, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9691698104143143 }, { + "base_success": 0.0, + "base_utility": 0.03280790150165558, "branch_car_at_8": 0.002354145050048828, "chart_id": "ms-3c326d05fec66e1d6bea", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11560099885411955, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0200461745262146, + "proposal_oracle_utility_gain_over_base_at_8": -0.012761726975440979, "seed": "278008524", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01769202947616577, + "selected_utility_gain_over_base_at_8": -0.015115872025489807, "selector_regret_at_8": 0.002354145050048828, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9799538254737854, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9823079705238342 }, { + "base_success": 0.0, + "base_utility": 0.1733371466398239, "branch_car_at_8": 0.0, "chart_id": "ms-4dbdc6ed61dccebbfe92", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12119924562800825, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176, "seed": "794181626", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.826662853360176, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-58950fed04d203b97ab9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3179396489681421, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1113371261", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.01641605794429779, "branch_car_at_8": 0.0, "chart_id": "ms-651bc0395d78d2819158", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022, "seed": "1483645336", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.9835839420557022, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, "chart_id": "ms-9fad82fd288e4296e354", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2176350935010736, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, "seed": "1176943004", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0689050555229187, "branch_car_at_8": 0.07749588787555695, "chart_id": "ms-b8bfdd3e6c31bd98082d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.23606164078349207, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08403940498828888, + "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178, "seed": "998761162", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006543517112731934, + "selected_utility_gain_over_base_at_8": -0.06236153841018677, "selector_regret_at_8": 0.07749588787555695, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9159605950117111, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.993456482887268 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-fc8084d5cd7da937fc02", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.05287821364799471, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1991414747", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.07482179999351501, "chart_id": "ms-3afd8959363e917591d4", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2497251397544613, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2396746575832367, + "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974, "seed": "75070641", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16485285758972168, + "selected_utility_gain_over_base_at_8": -0.03789877891540527, "selector_regret_at_8": 0.07482179999351501, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7603253424167633, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8351471424102783 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.183156132698059, "chart_id": "ms-3ff1aedb72f65a182b58", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07486610854656817, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1055567618", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8168438673019409, + "selected_utility_gain_over_base_at_8": -1.183156132698059, "selector_regret_at_8": 1.183156132698059, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.183156132698059 }, { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 1.8516457676887512, "chart_id": "ms-4048f8f6e047422e3c79", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2973966626262834, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087, "seed": "1055157354", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14835423231124878, + "selected_utility_gain_over_base_at_8": -0.02813686430454254, "selector_regret_at_8": 1.8516457676887512, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8516457676887512 }, { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.009524375200271606, "chart_id": "ms-5b26c33feed6992386b1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.08257819446483214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2435554713010788, + "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935, "seed": "1522272686", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2340310961008072, + "selected_utility_gain_over_base_at_8": 0.03926999866962433, "selector_regret_at_8": 0.009524375200271606, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7564445286989212, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7659689038991928 }, { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.5908347219228745, "chart_id": "ms-63ac245579b1188935ad", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16771630653845432, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8130142092704773, + "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624, "seed": "211245842", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22217948734760284, + "selected_utility_gain_over_base_at_8": -0.6166549175977707, "selector_regret_at_8": 0.5908347219228745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1869857907295227, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7778205126523972 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.007551252841949463, "chart_id": "ms-6c8d06bace98a28a8e4e", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.050940808914706356, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8216220736503601, + "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964, "seed": "1665739511", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8140708208084106, + "selected_utility_gain_over_base_at_8": -1.1859291791915894, "selector_regret_at_8": 0.007551252841949463, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.17837792634964, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.1859291791915894 }, { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.10820980370044708, "chart_id": "ms-72777024349c2aacd5e0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.030709639081801327, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.24107472598552704, + "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904, "seed": "256770874", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13286492228507996, + "selected_utility_gain_over_base_at_8": -0.7228831350803375, "selector_regret_at_8": 0.10820980370044708, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.758925274014473, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.86713507771492 }, { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 0.0, "chart_id": "ms-7fc5696cb3dc7b9bb9f7", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.20250855048512742, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5881698727607727, + "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979, "seed": "957101489", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5881698727607727, + "selected_utility_gain_over_base_at_8": 0.3842049241065979, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.4118301272392273, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.4118301272392273 }, { + "base_success": 0.0, + "base_utility": 0.8431589603424072, "branch_car_at_8": 1.7671720832586288, "chart_id": "ms-9a14552b7db695dee92c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.05415195874099398, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1568410396575928, "seed": "77147264", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23282791674137115, + "selected_utility_gain_over_base_at_8": -0.6103310436010361, "selector_regret_at_8": 1.7671720832586288, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7671720832586288 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.779732346534729, "chart_id": "ms-9f5d57fe2153fd267774", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08588083120555855, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1607773860", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.220267653465271, + "selected_utility_gain_over_base_at_8": -1.779732346534729, "selector_regret_at_8": 1.779732346534729, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.779732346534729 }, { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 0.5775325745344162, "chart_id": "ms-b0fee26e1db967f90b81", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13502516765102715, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8142459392547607, + "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605, "seed": "1358421325", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23671336472034454, + "selected_utility_gain_over_base_at_8": -0.5959889143705368, "selector_regret_at_8": 0.5775325745344162, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1857540607452393, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7632866352796555 }, { + "base_success": 0.0, + "base_utility": 0.8285209536552429, "branch_car_at_8": 0.0, "chart_id": "ms-bc2d3b425d9163a62dbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1508652338231993, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.171479046344757, "seed": "1271718007", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.171479046344757, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 0.3360496461391449, "chart_id": "ms-d1960368ef76bbb10669", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10129212810935397, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5778142213821411, + "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764, "seed": "416183106", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.24176457524299622, + "selected_utility_gain_over_base_at_8": 0.041438162326812744, "selector_regret_at_8": 0.3360496461391449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.4221857786178589, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7582354247570038 }, { + "base_success": 0.0, + "base_utility": 0.8125406503677368, "branch_car_at_8": 1.1865743398666382, "chart_id": "ms-efdc07c985935d871adf", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0922387871049479, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632, "seed": "706479600", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8134256601333618, + "selected_utility_gain_over_base_at_8": 0.000885009765625, "selector_regret_at_8": 1.1865743398666382, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.1865743398666382 }, { + "base_success": 0.0, + "base_utility": 0.037198856472969055, "branch_car_at_8": 0.016854479908943176, "chart_id": "ms-fd2387aeed9bfc9a0d08", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15779226512372985, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08992795646190643, + "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738, "seed": "1919198188", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07307347655296326, + "selected_utility_gain_over_base_at_8": 0.0358746200799942, "selector_regret_at_8": 0.016854479908943176, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9100720435380936, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9269265234470367 } ], "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4397163120567376, + "low": 0.18794326241134748, + "mean": 0.3085106382978724, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3085106382978724, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3125188536953243, + "low": 0.10153846153846154, + "mean": 0.2270286576168929, + "n": 5, + "num_samples": 500 + }, + "mean": 0.2270286576168929, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.3076923076923077, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.391304347826087, + "low": 0.17391304347826086, + "mean": 0.2753623188405797, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0788229198548414, + "low": 0.582660698415117, + "mean": 0.8136041222055977, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8136041222055977, + "per_seed": { + "1025740212": 0.528373658657074, + "1055157354": 0.17649109661579132, + "1055567618": 2.0, + "1113268940": 0.42231252789497375, + "1113371261": 2.0, + "1176943004": 0.05094052851200104, + "1189392917": 0.10841864347457886, + "1231780997": 0.7270061373710632, + "1271718007": 1.4142604768276215, + "1273247072": 0.15342965722084045, + "1358421325": 0.8327022790908813, + "1375492786": 2.0, + "141984615": 0.09381213908394177, + "1475440973": 2.0, + "1483645336": 0.01641605794429779, + "1522272686": 0.19476109743118286, + "1528740417": 2.0, + "1549324805": 1.3787474483251572, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.1463065892457962, + "1881223807": 0.0320269912481308, + "1919198188": 0.037198856472969055, + "1991414747": 2.0, + "1997090716": 0.13927479088306427, + "2015956957": 0.06711978986859321, + "2054899126": 2.0, + "211245842": 0.8388344049453735, + "2141802816": 0.07080724835395813, + "256770874": 0.8557480573654175, + "278008524": 0.10747186839580536, + "416183106": 0.20032641291618347, + "534717141": 2.0, + "551504635": 0.7239725589752197, + "565369392": 0.7997656464576721, + "565621826": 2.0, + "585904130": 0.07529209802548091, + "626486813": 0.5054400166263804, + "636680137": 2.0, + "706479600": 0.8125406503677368, + "750448296": 0.11153355985879898, + "75070641": 0.20275163650512695, + "767747424": 0.5475241877138615, + "77147264": 1.4215794801712036, + "794181626": 0.1733371466398239, + "957101489": 0.2039649486541748, + "998761162": 0.0689050555229187 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9041499242015874, + "low": 0.38615523747908764, + "mean": 0.6684181881773666, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6684181881773666, + "per_task": { + "LiftPegUpright-v1": 1.0669441471497219, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.6686872587754176, + "StackCube-v1": 0.8017999569574992 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9124907980254595, + "low": 0.5461467205060889, + "mean": 0.7161861676115381, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, "branch_car_at_8": { "macro_by_seed": { "ci": { @@ -855,6 +2139,186 @@ }, "n": 69 }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.922695035460993, + "low": 0.7351063829787235, + "mean": 0.8358156028368794, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8358156028368794, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.782608695652174, + "low": 0.5507246376811594, + "mean": 0.6666666666666666, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8615907955016442, + "low": 1.529367521088174, + "mean": 1.7062028111638965, + "n": 47, + "num_samples": 500 + }, + "mean": 1.7062028111638965, + "per_seed": { + "1025740212": 2.0, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 2.0, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1358421325": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.20044784247875214, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 2.0, + "1528740417": 2.0, + "1549324805": 1.4027505218982697, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 2.0, + "1881223807": 2.0, + "1919198188": 2.0, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.5298813670873642, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 2.0, + "278008524": 1.113811194896698, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 1.4266315797964733, + "565621826": 2.0, + "585904130": 0.23764699200789133, + "626486813": 0.6124684102833271, + "636680137": 2.0, + "706479600": 2.0, + "750448296": 0.23580759018659592, + "75070641": 2.0, + "767747424": 0.18863565661013126, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.6062410234109215, + "low": 1.1855228276572365, + "mean": 1.3999846732054932, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, "outcome_ptr_at_8": { "macro_by_seed": { "ci": { @@ -1035,134 +2499,1124 @@ }, "n": 69 }, - "selector_regret_at_8": { + "proposal_oracle_success_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5197332343231088, - "low": 0.16253579634643023, - "mean": 0.321408476649886, + "high": 0.5698581560283689, + "low": 0.29964539007092195, + "mean": 0.4404255319148936, "n": 47, "num_samples": 500 }, - "mean": 0.321408476649886, + "mean": 0.4404255319148936, "per_seed": { - "1025740212": 0.26188503205776215, - "1055157354": 1.8516457676887512, - "1055567618": 1.183156132698059, - "1113268940": 0.38470593094825745, - "1113371261": 0.0, - "1176943004": 0.0, - "1189392917": 0.0, - "1231780997": 0.4897426515817642, - "1271718007": 0.0, + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 1.0, "1273247072": 0.0, - "1358421325": 0.5775325745344162, - "1375492786": 0.0002854267368093133, - "141984615": 0.03316789741317431, - "1475440973": 0.0, - "1483645336": 0.0, - "1522272686": 0.009524375200271606, - "1528740417": 0.0, - "1549324805": 0.0034310482442378998, - "1607773860": 1.779732346534729, - "1665739511": 0.007551252841949463, - "1747178030": 0.0, - "1881223807": 0.020236089825630188, - "1919198188": 0.016854479908943176, - "1991414747": 0.0, - "1997090716": 1.9691698104143143, - "2015956957": 0.01770364921540022, - "2054899126": 0.0, - "211245842": 0.5908347219228745, - "2141802816": 0.07543906569480896, - "256770874": 0.10820980370044708, - "278008524": 0.0016456245793960989, - "416183106": 0.3360496461391449, - "534717141": 1.9274959191679955, - "551504635": 0.033253490924835205, - "565369392": 0.06443698455890019, - "565621826": 0.0, - "585904130": 0.043168901621053614, - "626486813": 0.018570441869087517, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, "636680137": 0.0, - "706479600": 1.1865743398666382, - "750448296": 0.07732226233929396, - "75070641": 0.07482179999351501, - "767747424": 0.015960504300892353, - "77147264": 1.868594542145729, - "794181626": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 1.0, + "794181626": 1.0, "957101489": 0.0, - "998761162": 0.07749588787555695 + "998761162": 0.0 } }, "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.5061151639620463, - "low": 0.07125234862643422, - "mean": 0.24495134526575732, + "high": 0.5408748114630468, + "low": 0.14666666666666667, + "mean": 0.34395173453996986, "n": 5, "num_samples": 500 }, - "mean": 0.24495134526575732, + "mean": 0.34395173453996986, "per_task": { - "LiftPegUpright-v1": 0.19493118425210318, + "LiftPegUpright-v1": 0.3333333333333333, "PickCube-v1": 0.0, - "PullCube-v1": 0.08066527944003396, - "PushCube-v1": 0.31651630768409145, - "StackCube-v1": 0.6326439549525579 + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.6923076923076923, + "StackCube-v1": 0.4 } }, - "metric": "selector_regret_at_8", + "metric": "proposal_oracle_success_at_8", "micro": { "confidence": 0.95, - "high": 0.378890726060701, - "low": 0.13165242864372398, - "mean": 0.2538634482703671, + "high": 0.5217391304347826, + "low": 0.2898550724637681, + "mean": 0.391304347826087, "n": 69, "num_samples": 500 }, "n": 69 }, - "support_gap_at_8": { + "proposal_oracle_success_gain_over_base_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.8930026109046337, - "low": 0.45526805628827577, - "mean": 0.6654373238655799, + "high": 0.27446808510638293, + "low": -0.010638297872340425, + "mean": 0.13191489361702127, "n": 47, "num_samples": 500 }, - "mean": 0.6654373238655799, + "mean": 0.13191489361702127, "per_seed": { - "1025740212": 1.5238737165927887, - "1055157354": 0.0, + "1025740212": 0.0, + "1055157354": 1.0, "1055567618": 0.0, - "1113268940": 1.3353878855705261, + "1113268940": 0.0, "1113371261": 0.0, - "1176943004": 0.0, - "1189392917": 0.0, - "1231780997": 1.2812588810920715, - "1271718007": 0.0, - "1273247072": 1.8672154396772385, - "1358421325": 1.1857540607452393, - "1375492786": 0.24180691444780678, - "141984615": 0.12070147196451823, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, "1475440973": 0.0, - "1483645336": 0.0, - "1522272686": 1.7564445286989212, + "1483645336": 1.0, + "1522272686": 0.0, "1528740417": 0.0, - "1549324805": 0.05998400536676248, + "1549324805": 0.0, "1607773860": 0.0, - "1665739511": 1.17837792634964, - "1747178030": 0.0, - "1881223807": 1.9486478716135025, - "1919198188": 1.9100720435380936, + "1665739511": -1.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, "1991414747": 0.0, - "1997090716": 0.0, - "2015956957": 0.07863824404776096, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.27076923076923076, + "low": 0.0, + "mean": 0.11692307692307694, + "n": 5, + "num_samples": 500 + }, + "mean": 0.11692307692307694, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.2 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.21739130434782608, + "low": 0.014492753623188406, + "mean": 0.11594202898550725, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2912854919257963, + "low": 0.8075891729536738, + "mean": 1.0507658735934828, + "n": 47, + "num_samples": 500 + }, + "mean": 1.0507658735934828, + "per_seed": { + "1025740212": 0.4761262834072113, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 0.6646121144294739, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.7187411189079285, + "1271718007": 2.0, + "1273247072": 0.13278456032276154, + "1358421325": 0.8142459392547607, + "1375492786": 0.0016440550098195672, + "141984615": 0.0797463705142339, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.2435554713010788, + "1528740417": 2.0, + "1549324805": 1.3427665165315072, + "1607773860": 2.0, + "1665739511": 0.8216220736503601, + "1747178030": 2.0, + "1881223807": 0.0513521283864975, + "1919198188": 0.08992795646190643, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.45124312303960323, + "2054899126": 2.0, + "211245842": 0.8130142092704773, + "2141802816": 0.12472955882549286, + "256770874": 0.24107472598552704, + "278008524": 0.011432985425926745, + "416183106": 0.5778142213821411, + "534717141": 2.0, + "551504635": 0.7250897884368896, + "565369392": 0.7852714310089747, + "565621826": 2.0, + "585904130": 0.05598785417775313, + "626486813": 0.5615248780231923, + "636680137": 0.012414653785526752, + "706479600": 2.0, + "750448296": 0.1069931760430336, + "75070641": 0.2396746575832367, + "767747424": 0.5703969299793243, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 0.5881698727607727, + "998761162": 0.08403940498828888 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2200616387067698, + "low": 0.4891751624122831, + "mean": 0.8746016368245654, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8746016368245654, + "per_task": { + "LiftPegUpright-v1": 1.0974282175302505, + "PickCube-v1": 0.13278456032276154, + "PullCube-v1": 0.6413554182680159, + "PushCube-v1": 1.406166712825115, + "StackCube-v1": 1.095273275176684 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.149046370209586, + "low": 0.73185485545917, + "mean": 0.9164160069709212, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5003876065769306, + "low": -0.0012803047515636864, + "mean": 0.23716175138788523, + "n": 47, + "num_samples": 500 + }, + "mean": 0.23716175138788523, + "per_seed": { + "1025740212": -0.05224737524986267, + "1055157354": 1.8235089033842087, + "1055567618": 0.0, + "1113268940": 0.24229958653450012, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": -0.008265018463134766, + "1271718007": 0.5857395231723785, + "1273247072": -0.02064509689807892, + "1358421325": -0.018456339836120605, + "1375492786": -1.9983559449901804, + "141984615": -0.01406576856970787, + "1475440973": 0.0, + "1483645336": 1.9835839420557022, + "1522272686": 0.048794373869895935, + "1528740417": 0.0, + "1549324805": -0.035980931793649994, + "1607773860": 0.0, + "1665739511": -1.17837792634964, + "1747178030": 1.8536934107542038, + "1881223807": 0.0193251371383667, + "1919198188": 0.05272909998893738, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": 0.38412333317101, + "2054899126": 0.0, + "211245842": -0.02582019567489624, + "2141802816": 0.05392231047153473, + "256770874": -0.6146733313798904, + "278008524": -0.09603888296987861, + "416183106": 0.37748780846595764, + "534717141": 0.0, + "551504635": 0.0011172294616699219, + "565369392": -0.014494215448697409, + "565621826": 0.0, + "585904130": -0.019304243847727776, + "626486813": 0.0560848613968119, + "636680137": -1.9875853462144732, + "706479600": 1.1874593496322632, + "750448296": -0.004540383815765381, + "75070641": 0.03692302107810974, + "767747424": 0.022872742265462875, + "77147264": 0.5784205198287964, + "794181626": 1.826662853360176, + "957101489": 0.3842049241065979, + "998761162": 0.015134349465370178 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4710757725513899, + "low": -0.00611102568926107, + "mean": 0.20618344864719879, + "n": 5, + "num_samples": 500 + }, + "mean": 0.20618344864719879, + "per_task": { + "LiftPegUpright-v1": 0.030484070380528767, + "PickCube-v1": -0.02064509689807892, + "PullCube-v1": -0.009874502515338142, + "PushCube-v1": 0.7374794540496973, + "StackCube-v1": 0.2934733182191849 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.37626325682588463, + "low": 0.0164921339938714, + "mean": 0.2002298393593831, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4219858156028369, + "low": 0.17978723404255317, + "mean": 0.2914893617021277, + "n": 47, + "num_samples": 500 + }, + "mean": 0.2914893617021277, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 1.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40307692307692305, + "low": 0.0796078431372549, + "mean": 0.24063348416289593, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24063348416289593, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.2647058823529412, + "PushCube-v1": 0.5384615384615384, + "StackCube-v1": 0.06666666666666667 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.37681159420289856, + "low": 0.18840579710144928, + "mean": 0.2753623188405797, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.12127659574468086, + "low": -0.16595744680851063, + "mean": -0.01702127659574468, + "n": 47, + "num_samples": 500 + }, + "mean": -0.01702127659574468, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": -1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": -1.0, + "1665739511": -1.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": -1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": -1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": -0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.12669683257918551, + "low": -0.08588235294117648, + "mean": 0.013604826546003018, + "n": 5, + "num_samples": 500 + }, + "mean": 0.013604826546003018, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.029411764705882353, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": -0.13333333333333333 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.11594202898550725, + "low": -0.10144927536231885, + "mean": 0.0, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9708039672328622, + "low": 0.5131759628765814, + "mean": 0.729357396943597, + "n": 47, + "num_samples": 500 + }, + "mean": 0.729357396943597, + "per_seed": { + "1025740212": 0.21424125134944916, + "1055157354": 0.14835423231124878, + "1055567618": 0.8168438673019409, + "1113268940": 0.27990618348121643, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.22899846732616425, + "1271718007": 2.0, + "1273247072": 0.13278456032276154, + "1358421325": 0.23671336472034454, + "1375492786": 0.001358628273010254, + "141984615": 0.04657847310105959, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.2340310961008072, + "1528740417": 2.0, + "1549324805": 1.3393354682872693, + "1607773860": 0.220267653465271, + "1665739511": 0.8140708208084106, + "1747178030": 2.0, + "1881223807": 0.03111603856086731, + "1919198188": 0.07307347655296326, + "1991414747": 2.0, + "1997090716": 0.03083018958568573, + "2015956957": 0.433539473824203, + "2054899126": 2.0, + "211245842": 0.22217948734760284, + "2141802816": 0.0492904931306839, + "256770874": 0.13286492228507996, + "278008524": 0.009787360846530646, + "416183106": 0.24176457524299622, + "534717141": 0.07250408083200455, + "551504635": 0.6918362975120544, + "565369392": 0.7208344464500746, + "565621826": 2.0, + "585904130": 0.012818952556699514, + "626486813": 0.5429544361541048, + "636680137": 0.012414653785526752, + "706479600": 0.8134256601333618, + "750448296": 0.029670913703739643, + "75070641": 0.16485285758972168, + "767747424": 0.554436425678432, + "77147264": 0.13140545785427094, + "794181626": 2.0, + "957101489": 0.5881698727607727, + "998761162": 0.006543517112731934 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9089970031332648, + "low": 0.35030357998435147, + "mean": 0.629650291558808, + "n": 5, + "num_samples": 500 + }, + "mean": 0.629650291558808, + "per_task": { + "LiftPegUpright-v1": 0.9024970332781473, + "PickCube-v1": 0.13278456032276154, + "PullCube-v1": 0.560690138827982, + "PushCube-v1": 1.0896504051410234, + "StackCube-v1": 0.4626293202241262 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.8693176263022115, + "low": 0.48426623860259366, + "mean": 0.6625525587005541, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.19248003160666863, + "low": -0.359282114742413, + "mean": -0.0842467252620007, + "n": 47, + "num_samples": 500 + }, + "mean": -0.0842467252620007, + "per_seed": { + "1025740212": -0.3141324073076248, + "1055157354": -0.02813686430454254, + "1055567618": -1.183156132698059, + "1113268940": -0.14240634441375732, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": -0.498007670044899, + "1271718007": 0.5857395231723785, + "1273247072": -0.02064509689807892, + "1358421325": -0.5959889143705368, + "1375492786": -1.9986413717269897, + "141984615": -0.04723366598288218, + "1475440973": 0.0, + "1483645336": 1.9835839420557022, + "1522272686": 0.03926999866962433, + "1528740417": 0.0, + "1549324805": -0.03941198003788789, + "1607773860": -1.779732346534729, + "1665739511": -1.1859291791915894, + "1747178030": 1.8536934107542038, + "1881223807": -0.0009109526872634888, + "1919198188": 0.0358746200799942, + "1991414747": 0.0, + "1997090716": -0.10844460129737854, + "2015956957": 0.3664196839556098, + "2054899126": 0.0, + "211245842": -0.6166549175977707, + "2141802816": -0.02151675522327423, + "256770874": -0.7228831350803375, + "278008524": -0.09768450754927471, + "416183106": 0.041438162326812744, + "534717141": -1.9274959191679955, + "551504635": -0.03213626146316528, + "565369392": -0.07893120000759761, + "565621826": 0.0, + "585904130": -0.06247314546878139, + "626486813": 0.037514419527724385, + "636680137": -1.9875853462144732, + "706479600": 0.000885009765625, + "750448296": -0.08186264615505934, + "75070641": -0.03789877891540527, + "767747424": 0.006912237964570522, + "77147264": -1.2901740223169327, + "794181626": 1.826662853360176, + "957101489": 0.3842049241065979, + "998761162": -0.06236153841018677 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.20158050865397423, + "low": -0.25449976120541307, + "mean": -0.038767896618558506, + "n": 5, + "num_samples": 500 + }, + "mean": -0.038767896618558506, + "per_task": { + "LiftPegUpright-v1": -0.1644471138715744, + "PickCube-v1": -0.02064509689807892, + "PullCube-v1": -0.0905397819553721, + "PushCube-v1": 0.4209631463656059, + "StackCube-v1": -0.339170636733373 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.14216782323901367, + "low": -0.2450303274233812, + "mean": -0.053633608910983996, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5197332343231088, + "low": 0.16253579634643023, + "mean": 0.321408476649886, + "n": 47, + "num_samples": 500 + }, + "mean": 0.321408476649886, + "per_seed": { + "1025740212": 0.26188503205776215, + "1055157354": 1.8516457676887512, + "1055567618": 1.183156132698059, + "1113268940": 0.38470593094825745, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.4897426515817642, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.5775325745344162, + "1375492786": 0.0002854267368093133, + "141984615": 0.03316789741317431, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.009524375200271606, + "1528740417": 0.0, + "1549324805": 0.0034310482442378998, + "1607773860": 1.779732346534729, + "1665739511": 0.007551252841949463, + "1747178030": 0.0, + "1881223807": 0.020236089825630188, + "1919198188": 0.016854479908943176, + "1991414747": 0.0, + "1997090716": 1.9691698104143143, + "2015956957": 0.01770364921540022, + "2054899126": 0.0, + "211245842": 0.5908347219228745, + "2141802816": 0.07543906569480896, + "256770874": 0.10820980370044708, + "278008524": 0.0016456245793960989, + "416183106": 0.3360496461391449, + "534717141": 1.9274959191679955, + "551504635": 0.033253490924835205, + "565369392": 0.06443698455890019, + "565621826": 0.0, + "585904130": 0.043168901621053614, + "626486813": 0.018570441869087517, + "636680137": 0.0, + "706479600": 1.1865743398666382, + "750448296": 0.07732226233929396, + "75070641": 0.07482179999351501, + "767747424": 0.015960504300892353, + "77147264": 1.868594542145729, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.07749588787555695 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5061151639620463, + "low": 0.07125234862643422, + "mean": 0.24495134526575732, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24495134526575732, + "per_task": { + "LiftPegUpright-v1": 0.19493118425210318, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08066527944003396, + "PushCube-v1": 0.31651630768409145, + "StackCube-v1": 0.6326439549525579 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.378890726060701, + "low": 0.13165242864372398, + "mean": 0.2538634482703671, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2553191489361702, + "low": 0.0425531914893617, + "mean": 0.14893617021276595, + "n": 47, + "num_samples": 500 + }, + "mean": 0.14893617021276595, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2615384615384615, + "low": 0.011764705882352941, + "mean": 0.1033182503770739, + "n": 5, + "num_samples": 500 + }, + "mean": 0.1033182503770739, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.029411764705882353, + "PushCube-v1": 0.15384615384615385, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.18840579710144928, + "low": 0.043478260869565216, + "mean": 0.11594202898550725, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5531914893617021, + "low": 0.2695035460992908, + "mean": 0.40070921985815605, + "n": 47, + "num_samples": 500 + }, + "mean": 0.40070921985815605, + "per_seed": { + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7948717948717949, + "low": 0.21683257918552035, + "mean": 0.5266365007541478, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5266365007541478, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.058823529411764705, + "PushCube-v1": 0.3076923076923077, + "StackCube-v1": 0.6 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.391304347826087, + "low": 0.17391304347826086, + "mean": 0.2898550724637681, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6950354609929079, + "low": 0.4113475177304965, + "mean": 0.549645390070922, + "n": 47, + "num_samples": 500 + }, + "mean": 0.549645390070922, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9066666666666666, + "low": 0.3122171945701358, + "mean": 0.6299547511312216, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6299547511312216, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.08823529411764706, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.9333333333333333 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5217391304347826, + "low": 0.2753623188405797, + "mean": 0.4057971014492754, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8930026109046337, + "low": 0.45526805628827577, + "mean": 0.6654373238655799, + "n": 47, + "num_samples": 500 + }, + "mean": 0.6654373238655799, + "per_seed": { + "1025740212": 1.5238737165927887, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 1.3353878855705261, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.2812588810920715, + "1271718007": 0.0, + "1273247072": 1.8672154396772385, + "1358421325": 1.1857540607452393, + "1375492786": 0.24180691444780678, + "141984615": 0.12070147196451823, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.7564445286989212, + "1528740417": 0.0, + "1549324805": 0.05998400536676248, + "1607773860": 0.0, + "1665739511": 1.17837792634964, + "1747178030": 0.0, + "1881223807": 1.9486478716135025, + "1919198188": 1.9100720435380936, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.07863824404776096, "2054899126": 0.0, "211245842": 1.1869857907295227, "2141802816": 1.8752704411745071, @@ -1214,6 +3668,96 @@ "num_samples": 500 }, "n": 69 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.235173605110724, + "low": 0.7539350982827253, + "mean": 0.9865234781741099, + "n": 47, + "num_samples": 500 + }, + "mean": 0.9865234781741099, + "per_seed": { + "1025740212": 1.7857587486505508, + "1055157354": 1.8516457676887512, + "1055567618": 1.183156132698059, + "1113268940": 1.7200938165187836, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.7710015326738358, + "1271718007": 0.0, + "1273247072": 1.8672154396772385, + "1358421325": 1.7632866352796555, + "1375492786": 0.2420923411846161, + "141984615": 0.15386936937769255, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.7659689038991928, + "1528740417": 0.0, + "1549324805": 0.06341505361100037, + "1607773860": 1.779732346534729, + "1665739511": 1.1859291791915894, + "1747178030": 0.0, + "1881223807": 1.9688839614391327, + "1919198188": 1.9269265234470367, + "1991414747": 0.0, + "1997090716": 1.9691698104143143, + "2015956957": 0.09634189326316119, + "2054899126": 0.0, + "211245842": 1.7778205126523972, + "2141802816": 1.950709506869316, + "256770874": 1.86713507771492, + "278008524": 1.1040238340501674, + "416183106": 1.7582354247570038, + "534717141": 1.9274959191679955, + "551504635": 1.3081637024879456, + "565369392": 0.7057971333463987, + "565621826": 0.0, + "585904130": 0.2248280394511918, + "626486813": 0.08356922247912735, + "636680137": 1.9875853462144732, + "706479600": 1.1865743398666382, + "750448296": 0.20613667648285627, + "75070641": 1.8351471424102783, + "767747424": 0.0750129884108901, + "77147264": 1.868594542145729, + "794181626": 0.0, + "957101489": 1.4118301272392273, + "998761162": 1.993456482887268 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6098733187333132, + "low": 0.6546314741212763, + "mean": 1.1375168759382237, + "n": 5, + "num_samples": 500 + }, + "mean": 1.1375168759382237, + "per_task": { + "LiftPegUpright-v1": 1.0975029667218525, + "PickCube-v1": 1.8672154396772385, + "PullCube-v1": 0.2751456986571772, + "PushCube-v1": 0.9103495948589765, + "StackCube-v1": 1.5373706797758737 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9371206429449545, + "low": 0.5655091602922134, + "mean": 0.763801332234162, + "n": 69, + "num_samples": 500 + }, + "n": 69 } }, "thresholds": [ diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_seed.json index 7aa274b859101bdbf1034124e2dc74e597f7d99d..c1f2e5ecf7f8d78fe849abbfd7ee49e5d29eddae 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_seed.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_seed.json @@ -1,331 +1,1083 @@ { "1025740212": { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.26188503205776215, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08963387331564551, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4761262834072113, + "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21424125134944916, + "selected_utility_gain_over_base_at_8": -0.3141324073076248, "selector_regret_at_8": 0.26188503205776215, - "support_gap_at_8": 1.5238737165927887 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5238737165927887, + "total_car_to_hidden_at_8": 1.7857587486505508 }, "1055157354": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 1.8516457676887512, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2973966626262834, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14835423231124878, + "selected_utility_gain_over_base_at_8": -0.02813686430454254, "selector_regret_at_8": 1.8516457676887512, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8516457676887512 }, "1055567618": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.183156132698059, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07486610854656817, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8168438673019409, + "selected_utility_gain_over_base_at_8": -1.183156132698059, "selector_regret_at_8": 1.183156132698059, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.183156132698059 }, "1113268940": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.38470593094825745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1616344352768096, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6646121144294739, + "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.27990618348121643, + "selected_utility_gain_over_base_at_8": -0.14240634441375732, "selector_regret_at_8": 0.38470593094825745, - "support_gap_at_8": 1.3353878855705261 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3353878855705261, + "total_car_to_hidden_at_8": 1.7200938165187836 }, "1113371261": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3179396489681421, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1176943004": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2176350935010736, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1231780997": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.4897426515817642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18322944379343714, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7187411189079285, + "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22899846732616425, + "selected_utility_gain_over_base_at_8": -0.498007670044899, "selector_regret_at_8": 0.4897426515817642, - "support_gap_at_8": 1.2812588810920715 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2812588810920715, + "total_car_to_hidden_at_8": 1.7710015326738358 }, "1271718007": { + "base_success": 0.5, + "base_utility": 1.4142604768276215, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.07543261691159965, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5857395231723785, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5857395231723785, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.8672154396772385 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 }, "1358421325": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 0.5775325745344162, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13502516765102715, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8142459392547607, + "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23671336472034454, + "selected_utility_gain_over_base_at_8": -0.5959889143705368, "selector_regret_at_8": 0.5775325745344162, - "support_gap_at_8": 1.1857540607452393 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1857540607452393, + "total_car_to_hidden_at_8": 1.7632866352796555 }, "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002854267368093133, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, "selector_regret_at_8": 0.0002854267368093133, - "support_gap_at_8": 0.24180691444780678 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24180691444780678, + "total_car_to_hidden_at_8": 0.2420923411846161 }, "141984615": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, "branch_car_at_8": 0.03316789741317431, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.10926310602682258, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0797463705142339, + "proposal_oracle_utility_gain_over_base_at_8": -0.01406576856970787, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04657847310105959, + "selected_utility_gain_over_base_at_8": -0.04723366598288218, "selector_regret_at_8": 0.03316789741317431, - "support_gap_at_8": 0.12070147196451823 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12070147196451823, + "total_car_to_hidden_at_8": 0.15386936937769255 }, "1475440973": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.09930428958964055, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1483645336": { + "base_success": 0.0, + "base_utility": 0.01641605794429779, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.9835839420557022, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1522272686": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.009524375200271606, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.08257819446483214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2435554713010788, + "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2340310961008072, + "selected_utility_gain_over_base_at_8": 0.03926999866962433, "selector_regret_at_8": 0.009524375200271606, - "support_gap_at_8": 1.7564445286989212 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7564445286989212, + "total_car_to_hidden_at_8": 1.7659689038991928 }, "1528740417": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.015990866806051107, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1549324805": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, "branch_car_at_8": 0.0034310482442378998, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19382954770507574, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.3427665165315072, + "proposal_oracle_utility_gain_over_base_at_8": -0.035980931793649994, + "selected_success_at_8": 0.6666666666666666, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.3393354682872693, + "selected_utility_gain_over_base_at_8": -0.03941198003788789, "selector_regret_at_8": 0.0034310482442378998, - "support_gap_at_8": 0.05998400536676248 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05998400536676248, + "total_car_to_hidden_at_8": 0.06341505361100037 }, "1607773860": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.779732346534729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08588083120555855, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.220267653465271, + "selected_utility_gain_over_base_at_8": -1.779732346534729, "selector_regret_at_8": 1.779732346534729, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.779732346534729 }, "1665739511": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.007551252841949463, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.050940808914706356, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8216220736503601, + "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8140708208084106, + "selected_utility_gain_over_base_at_8": -1.1859291791915894, "selector_regret_at_8": 0.007551252841949463, - "support_gap_at_8": 1.17837792634964 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.17837792634964, + "total_car_to_hidden_at_8": 1.1859291791915894 }, "1747178030": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15825371494561297, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8536934107542038, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.020236089825630188, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, "selector_regret_at_8": 0.020236089825630188, - "support_gap_at_8": 1.9486478716135025 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9486478716135025, + "total_car_to_hidden_at_8": 1.9688839614391327 }, "1919198188": { + "base_success": 0.0, + "base_utility": 0.037198856472969055, "branch_car_at_8": 0.016854479908943176, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15779226512372985, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08992795646190643, + "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07307347655296326, + "selected_utility_gain_over_base_at_8": 0.0358746200799942, "selector_regret_at_8": 0.016854479908943176, - "support_gap_at_8": 1.9100720435380936 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9100720435380936, + "total_car_to_hidden_at_8": 1.9269265234470367 }, "1991414747": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.05287821364799471, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 1.9691698104143143, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, "selector_regret_at_8": 1.9691698104143143, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9691698104143143 }, "2015956957": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, "branch_car_at_8": 0.01770364921540022, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.18664994005668376, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.45124312303960323, + "proposal_oracle_utility_gain_over_base_at_8": 0.38412333317101, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.433539473824203, + "selected_utility_gain_over_base_at_8": 0.3664196839556098, "selector_regret_at_8": 0.01770364921540022, - "support_gap_at_8": 0.07863824404776096 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07863824404776096, + "total_car_to_hidden_at_8": 0.09634189326316119 }, "2054899126": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "211245842": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.5908347219228745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16771630653845432, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8130142092704773, + "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22217948734760284, + "selected_utility_gain_over_base_at_8": -0.6166549175977707, "selector_regret_at_8": 0.5908347219228745, - "support_gap_at_8": 1.1869857907295227 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1869857907295227, + "total_car_to_hidden_at_8": 1.7778205126523972 }, "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.07543906569480896, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, "selector_regret_at_8": 0.07543906569480896, - "support_gap_at_8": 1.8752704411745071 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8752704411745071, + "total_car_to_hidden_at_8": 1.950709506869316 }, "256770874": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.10820980370044708, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.030709639081801327, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.24107472598552704, + "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13286492228507996, + "selected_utility_gain_over_base_at_8": -0.7228831350803375, "selector_regret_at_8": 0.10820980370044708, - "support_gap_at_8": 1.758925274014473 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.758925274014473, + "total_car_to_hidden_at_8": 1.86713507771492 }, "278008524": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, "branch_car_at_8": 0.0016456245793960989, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11925245416072927, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.011432985425926745, + "proposal_oracle_utility_gain_over_base_at_8": -0.09603888296987861, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.009787360846530646, + "selected_utility_gain_over_base_at_8": -0.09768450754927471, "selector_regret_at_8": 0.0016456245793960989, - "support_gap_at_8": 1.1023782094707713 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.1023782094707713, + "total_car_to_hidden_at_8": 1.1040238340501674 }, "416183106": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 0.3360496461391449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10129212810935397, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5778142213821411, + "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.24176457524299622, + "selected_utility_gain_over_base_at_8": 0.041438162326812744, "selector_regret_at_8": 0.3360496461391449, - "support_gap_at_8": 1.4221857786178589 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4221857786178589, + "total_car_to_hidden_at_8": 1.7582354247570038 }, "534717141": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9274959191679955, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.6159370955924921, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07250408083200455, + "selected_utility_gain_over_base_at_8": -1.9274959191679955, "selector_regret_at_8": 1.9274959191679955, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9274959191679955 }, "551504635": { + "base_success": 0.0, + "base_utility": 0.7239725589752197, "branch_car_at_8": 0.033253490924835205, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10932818664818449, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7250897884368896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6918362975120544, + "selected_utility_gain_over_base_at_8": -0.03213626146316528, "selector_regret_at_8": 0.033253490924835205, - "support_gap_at_8": 1.2749102115631104 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2749102115631104, + "total_car_to_hidden_at_8": 1.3081637024879456 }, "565369392": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, "branch_car_at_8": 0.06443698455890019, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22308845281795123, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7852714310089747, + "proposal_oracle_utility_gain_over_base_at_8": -0.014494215448697409, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7208344464500746, + "selected_utility_gain_over_base_at_8": -0.07893120000759761, "selector_regret_at_8": 0.06443698455890019, - "support_gap_at_8": 0.6413601487874985 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.6413601487874985, + "total_car_to_hidden_at_8": 0.7057971333463987 }, "565621826": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "585904130": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, "branch_car_at_8": 0.043168901621053614, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.10391809443539522, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05598785417775313, + "proposal_oracle_utility_gain_over_base_at_8": -0.019304243847727776, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012818952556699514, + "selected_utility_gain_over_base_at_8": -0.06247314546878139, "selector_regret_at_8": 0.043168901621053614, - "support_gap_at_8": 0.1816591378301382 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1816591378301382, + "total_car_to_hidden_at_8": 0.2248280394511918 }, "626486813": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, "branch_car_at_8": 0.018570441869087517, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.16162412565990822, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5615248780231923, + "proposal_oracle_utility_gain_over_base_at_8": 0.0560848613968119, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5429544361541048, + "selected_utility_gain_over_base_at_8": 0.037514419527724385, "selector_regret_at_8": 0.018570441869087517, - "support_gap_at_8": 0.0801479306537658 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0801479306537658, + "total_car_to_hidden_at_8": 0.08356922247912735 }, "636680137": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1699944666253151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012414653785526752, + "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.012414653785526752, + "selected_utility_gain_over_base_at_8": -1.9875853462144732, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.9875853462144732 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9875853462144732, + "total_car_to_hidden_at_8": 1.9875853462144732 }, "706479600": { + "base_success": 0.0, + "base_utility": 0.8125406503677368, "branch_car_at_8": 1.1865743398666382, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0922387871049479, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8134256601333618, + "selected_utility_gain_over_base_at_8": 0.000885009765625, "selector_regret_at_8": 1.1865743398666382, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.1865743398666382 }, "750448296": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, "branch_car_at_8": 0.07732226233929396, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.14360585133877676, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1069931760430336, + "proposal_oracle_utility_gain_over_base_at_8": -0.004540383815765381, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.029670913703739643, + "selected_utility_gain_over_base_at_8": -0.08186264615505934, "selector_regret_at_8": 0.07732226233929396, - "support_gap_at_8": 0.12881441414356232 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12881441414356232, + "total_car_to_hidden_at_8": 0.20613667648285627 }, "75070641": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.07482179999351501, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2497251397544613, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2396746575832367, + "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16485285758972168, + "selected_utility_gain_over_base_at_8": -0.03789877891540527, "selector_regret_at_8": 0.07482179999351501, - "support_gap_at_8": 1.7603253424167633 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7603253424167633, + "total_car_to_hidden_at_8": 1.8351471424102783 }, "767747424": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, "branch_car_at_8": 0.015960504300892353, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, "outcome_ptr_at_8": 0.25, "pairwise_causal_calibration_ece": 0.21088094062197144, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5703969299793243, + "proposal_oracle_utility_gain_over_base_at_8": 0.022872742265462875, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.554436425678432, + "selected_utility_gain_over_base_at_8": 0.006912237964570522, "selector_regret_at_8": 0.015960504300892353, - "support_gap_at_8": 0.05905248410999775 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05905248410999775, + "total_car_to_hidden_at_8": 0.0750129884108901 }, "77147264": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, "branch_car_at_8": 1.868594542145729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.06512707818376468, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.5, + "selected_utility_at_8": 0.13140545785427094, + "selected_utility_gain_over_base_at_8": -1.2901740223169327, "selector_regret_at_8": 1.868594542145729, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.868594542145729 }, "794181626": { + "base_success": 0.0, + "base_utility": 0.1733371466398239, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12119924562800825, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.826662853360176, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "957101489": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.20250855048512742, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5881698727607727, + "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5881698727607727, + "selected_utility_gain_over_base_at_8": 0.3842049241065979, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.4118301272392273 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4118301272392273, + "total_car_to_hidden_at_8": 1.4118301272392273 }, "998761162": { + "base_success": 0.0, + "base_utility": 0.0689050555229187, "branch_car_at_8": 0.07749588787555695, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.23606164078349207, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08403940498828888, + "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006543517112731934, + "selected_utility_gain_over_base_at_8": -0.06236153841018677, "selector_regret_at_8": 0.07749588787555695, - "support_gap_at_8": 1.9159605950117111 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9159605950117111, + "total_car_to_hidden_at_8": 1.993456482887268 } } diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_task.json index cbccfc6668c62b2df6034d73d0f4b87377341965..221883cb828eead25cb9dc4e96dca084a508ea33 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_task.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/metrics_by_task.json @@ -1,37 +1,117 @@ { "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.0669441471497219, "branch_car_at_8": 0.19493118425210318, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.1098535159049614, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.0974282175302505, + "proposal_oracle_utility_gain_over_base_at_8": 0.030484070380528767, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.9024970332781473, + "selected_utility_gain_over_base_at_8": -0.1644471138715744, "selector_regret_at_8": 0.19493118425210318, - "support_gap_at_8": 0.9025717824697495 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.9025717824697495, + "total_car_to_hidden_at_8": 1.0975029667218525 }, "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.8672154396772385 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 }, "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, "branch_car_at_8": 0.08066527944003396, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, "outcome_ptr_at_8": 0.3235294117647059, "pairwise_causal_calibration_ece": 0.17256325415222457, + "proposal_oracle_success_at_8": 0.29411764705882354, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6413554182680159, + "proposal_oracle_utility_gain_over_base_at_8": -0.009874502515338142, + "selected_success_at_8": 0.2647058823529412, + "selected_success_gain_over_base_at_8": -0.029411764705882353, + "selected_utility_at_8": 0.560690138827982, + "selected_utility_gain_over_base_at_8": -0.0905397819553721, "selector_regret_at_8": 0.08066527944003396, - "support_gap_at_8": 0.1962626721634639 + "success_selector_gap_at_8": 0.029411764705882353, + "success_support_gap_at_8": 0.058823529411764705, + "success_total_car_to_hidden_at_8": 0.08823529411764706, + "support_gap_at_8": 0.1962626721634639, + "total_car_to_hidden_at_8": 0.2751456986571772 }, "PushCube-v1": { + "base_success": 0.3076923076923077, + "base_utility": 0.6686872587754176, "branch_car_at_8": 0.31651630768409145, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6153846153846154, "pairwise_causal_calibration_ece": 0.13663407835877941, + "proposal_oracle_success_at_8": 0.6923076923076923, + "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464, + "proposal_oracle_utility_at_8": 1.406166712825115, + "proposal_oracle_utility_gain_over_base_at_8": 0.7374794540496973, + "selected_success_at_8": 0.5384615384615384, + "selected_success_gain_over_base_at_8": 0.23076923076923078, + "selected_utility_at_8": 1.0896504051410234, + "selected_utility_gain_over_base_at_8": 0.4209631463656059, "selector_regret_at_8": 0.31651630768409145, - "support_gap_at_8": 0.593833287174885 + "success_selector_gap_at_8": 0.15384615384615385, + "success_support_gap_at_8": 0.3076923076923077, + "success_total_car_to_hidden_at_8": 0.46153846153846156, + "support_gap_at_8": 0.593833287174885, + "total_car_to_hidden_at_8": 0.9103495948589765 }, "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8017999569574992, "branch_car_at_8": 0.6326439549525579, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.128912518811403, + "proposal_oracle_success_at_8": 0.4, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 1.095273275176684, + "proposal_oracle_utility_gain_over_base_at_8": 0.2934733182191849, + "selected_success_at_8": 0.06666666666666667, + "selected_success_gain_over_base_at_8": -0.13333333333333333, + "selected_utility_at_8": 0.4626293202241262, + "selected_utility_gain_over_base_at_8": -0.339170636733373, "selector_regret_at_8": 0.6326439549525579, - "support_gap_at_8": 0.904726724823316 + "success_selector_gap_at_8": 0.3333333333333333, + "success_support_gap_at_8": 0.6, + "success_total_car_to_hidden_at_8": 0.9333333333333333, + "support_gap_at_8": 0.904726724823316, + "total_car_to_hidden_at_8": 1.5373706797758737 } } diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/report.md b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/report.md index 540301281098bb18073504e697766bd1c9bd9357..a5c073814af85af83770a02bd423f55d45a50471 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/report.md +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/report.md @@ -4,8 +4,24 @@ K: `8` | Metric | N | Micro mean | 95% CI | Task macro | Seed macro | | --- | ---: | ---: | ---: | ---: | ---: | +| base_success | 69 | 0.2754 | [0.1739, 0.3913] | 0.2270 | 0.3085 | +| base_utility | 69 | 0.7162 | [0.5461, 0.9125] | 0.6684 | 0.8136 | | branch_car_at_8 | 69 | 0.2539 | [0.1317, 0.3789] | 0.2450 | 0.3214 | +| hidden_chart_oracle_success_at_8 | 69 | 0.6667 | [0.5507, 0.7826] | 0.8647 | 0.8358 | +| hidden_chart_oracle_utility_at_8 | 69 | 1.4000 | [1.1855, 1.6062] | 1.7565 | 1.7062 | | outcome_ptr_at_8 | 69 | 0.4348 | [0.3188, 0.5507] | 0.3744 | 0.4631 | | pairwise_causal_calibration_ece | 69 | 0.1505 | [0.1261, 0.1764] | 0.1397 | 0.1459 | +| proposal_oracle_success_at_8 | 69 | 0.3913 | [0.2899, 0.5217] | 0.3440 | 0.4404 | +| proposal_oracle_success_gain_over_base_at_8 | 69 | 0.1159 | [0.0145, 0.2174] | 0.1169 | 0.1319 | +| proposal_oracle_utility_at_8 | 69 | 0.9164 | [0.7319, 1.1490] | 0.8746 | 1.0508 | +| proposal_oracle_utility_gain_over_base_at_8 | 69 | 0.2002 | [0.0165, 0.3763] | 0.2062 | 0.2372 | +| selected_success_at_8 | 69 | 0.2754 | [0.1884, 0.3768] | 0.2406 | 0.2915 | +| selected_success_gain_over_base_at_8 | 69 | 0.0000 | [-0.1014, 0.1159] | 0.0136 | -0.0170 | +| selected_utility_at_8 | 69 | 0.6626 | [0.4843, 0.8693] | 0.6297 | 0.7294 | +| selected_utility_gain_over_base_at_8 | 69 | -0.0536 | [-0.2450, 0.1422] | -0.0388 | -0.0842 | | selector_regret_at_8 | 69 | 0.2539 | [0.1317, 0.3789] | 0.2450 | 0.3214 | +| success_selector_gap_at_8 | 69 | 0.1159 | [0.0435, 0.1884] | 0.1033 | 0.1489 | +| success_support_gap_at_8 | 69 | 0.2899 | [0.1739, 0.3913] | 0.5266 | 0.4007 | +| success_total_car_to_hidden_at_8 | 69 | 0.4058 | [0.2754, 0.5217] | 0.6300 | 0.5496 | | support_gap_at_8 | 69 | 0.5108 | [0.3299, 0.6784] | 0.8929 | 0.6654 | +| total_car_to_hidden_at_8 | 69 | 0.7638 | [0.5655, 0.9371] | 1.1375 | 0.9865 | diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/table.tex b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/table.tex index f5efb9b0f57659b074f7a650354d3fe5bb3e3834..8ef54f8e7b0c023991b43b48494640a2c662ca34 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/table.tex +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/measured_metrics/table.tex @@ -3,10 +3,26 @@ \toprule Metric & N & Micro mean & CI low & CI high \\ \midrule +base\_success & 69 & 0.2754 & 0.1739 & 0.3913 \\ +base\_utility & 69 & 0.7162 & 0.5461 & 0.9125 \\ branch\_car\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\ +hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\ outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3188 & 0.5507 \\ pairwise\_causal\_calibration\_ece & 69 & 0.1505 & 0.1261 & 0.1764 \\ +proposal\_oracle\_success\_at\_8 & 69 & 0.3913 & 0.2899 & 0.5217 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1159 & 0.0145 & 0.2174 \\ +proposal\_oracle\_utility\_at\_8 & 69 & 0.9164 & 0.7319 & 1.1490 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.2002 & 0.0165 & 0.3763 \\ +selected\_success\_at\_8 & 69 & 0.2754 & 0.1884 & 0.3768 \\ +selected\_success\_gain\_over\_base\_at\_8 & 69 & 0.0000 & -0.1014 & 0.1159 \\ +selected\_utility\_at\_8 & 69 & 0.6626 & 0.4843 & 0.8693 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.0536 & -0.2450 & 0.1422 \\ selector\_regret\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\ +success\_selector\_gap\_at\_8 & 69 & 0.1159 & 0.0435 & 0.1884 \\ +success\_support\_gap\_at\_8 & 69 & 0.2899 & 0.1739 & 0.3913 \\ +success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\ support\_gap\_at\_8 & 69 & 0.5108 & 0.3299 & 0.6784 \\ +total\_car\_to\_hidden\_at\_8 & 69 & 0.7638 & 0.5655 & 0.9371 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/metrics.json b/workspace/runs/ctt_residual_rollout_val69_seed0/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3808ad56583dd15e2c59439e49dceecbdf15616d --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/metrics.json @@ -0,0 +1,1918 @@ +{ + "checkpoint": "runs/ctt_residual_full_seed0/model.pt", + "k": 8, + "measured_metric_summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4397163120567376, + "low": 0.18794326241134748, + "mean": 0.3085106382978724, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3085106382978724, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3125188536953243, + "low": 0.10153846153846154, + "mean": 0.2270286576168929, + "n": 5, + "num_samples": 500 + }, + "mean": 0.2270286576168929, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.3076923076923077, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.391304347826087, + "low": 0.17391304347826086, + "mean": 0.2753623188405797, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0788229198548414, + "low": 0.582660698415117, + "mean": 0.8136041222055977, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8136041222055977, + "per_seed": { + "1025740212": 0.528373658657074, + "1055157354": 0.17649109661579132, + "1055567618": 2.0, + "1113268940": 0.42231252789497375, + "1113371261": 2.0, + "1176943004": 0.05094052851200104, + "1189392917": 0.10841864347457886, + "1231780997": 0.7270061373710632, + "1271718007": 1.4142604768276215, + "1273247072": 0.15342965722084045, + "1358421325": 0.8327022790908813, + "1375492786": 2.0, + "141984615": 0.09381213908394177, + "1475440973": 2.0, + "1483645336": 0.01641605794429779, + "1522272686": 0.19476109743118286, + "1528740417": 2.0, + "1549324805": 1.3787474483251572, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.1463065892457962, + "1881223807": 0.0320269912481308, + "1919198188": 0.037198856472969055, + "1991414747": 2.0, + "1997090716": 0.13927479088306427, + "2015956957": 0.06711978986859321, + "2054899126": 2.0, + "211245842": 0.8388344049453735, + "2141802816": 0.07080724835395813, + "256770874": 0.8557480573654175, + "278008524": 0.10747186839580536, + "416183106": 0.20032641291618347, + "534717141": 2.0, + "551504635": 0.7239725589752197, + "565369392": 0.7997656464576721, + "565621826": 2.0, + "585904130": 0.07529209802548091, + "626486813": 0.5054400166263804, + "636680137": 2.0, + "706479600": 0.8125406503677368, + "750448296": 0.11153355985879898, + "75070641": 0.20275163650512695, + "767747424": 0.5475241877138615, + "77147264": 1.4215794801712036, + "794181626": 0.1733371466398239, + "957101489": 0.2039649486541748, + "998761162": 0.0689050555229187 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9041499242015874, + "low": 0.38615523747908764, + "mean": 0.6684181881773666, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6684181881773666, + "per_task": { + "LiftPegUpright-v1": 1.0669441471497219, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.6686872587754176, + "StackCube-v1": 0.8017999569574992 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9124907980254595, + "low": 0.5461467205060889, + "mean": 0.7161861676115381, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "branch_car_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5197332343231088, + "low": 0.16253579634643023, + "mean": 0.321408476649886, + "n": 47, + "num_samples": 500 + }, + "mean": 0.321408476649886, + "per_seed": { + "1025740212": 0.26188503205776215, + "1055157354": 1.8516457676887512, + "1055567618": 1.183156132698059, + "1113268940": 0.38470593094825745, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.4897426515817642, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.5775325745344162, + "1375492786": 0.0002854267368093133, + "141984615": 0.03316789741317431, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.009524375200271606, + "1528740417": 0.0, + "1549324805": 0.0034310482442378998, + "1607773860": 1.779732346534729, + "1665739511": 0.007551252841949463, + "1747178030": 0.0, + "1881223807": 0.020236089825630188, + "1919198188": 0.016854479908943176, + "1991414747": 0.0, + "1997090716": 1.9691698104143143, + "2015956957": 0.01770364921540022, + "2054899126": 0.0, + "211245842": 0.5908347219228745, + "2141802816": 0.07543906569480896, + "256770874": 0.10820980370044708, + "278008524": 0.0016456245793960989, + "416183106": 0.3360496461391449, + "534717141": 1.9274959191679955, + "551504635": 0.033253490924835205, + "565369392": 0.06443698455890019, + "565621826": 0.0, + "585904130": 0.043168901621053614, + "626486813": 0.018570441869087517, + "636680137": 0.0, + "706479600": 1.1865743398666382, + "750448296": 0.07732226233929396, + "75070641": 0.07482179999351501, + "767747424": 0.015960504300892353, + "77147264": 1.868594542145729, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.07749588787555695 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5061151639620463, + "low": 0.07125234862643422, + "mean": 0.24495134526575732, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24495134526575732, + "per_task": { + "LiftPegUpright-v1": 0.19493118425210318, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08066527944003396, + "PushCube-v1": 0.31651630768409145, + "StackCube-v1": 0.6326439549525579 + } + }, + "metric": "branch_car_at_8", + "micro": { + "confidence": 0.95, + "high": 0.378890726060701, + "low": 0.13165242864372398, + "mean": 0.2538634482703671, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.922695035460993, + "low": 0.7351063829787235, + "mean": 0.8358156028368794, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8358156028368794, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.782608695652174, + "low": 0.5507246376811594, + "mean": 0.6666666666666666, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8615907955016442, + "low": 1.529367521088174, + "mean": 1.7062028111638965, + "n": 47, + "num_samples": 500 + }, + "mean": 1.7062028111638965, + "per_seed": { + "1025740212": 2.0, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 2.0, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1358421325": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.20044784247875214, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 2.0, + "1528740417": 2.0, + "1549324805": 1.4027505218982697, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 2.0, + "1881223807": 2.0, + "1919198188": 2.0, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.5298813670873642, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 2.0, + "278008524": 1.113811194896698, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 1.4266315797964733, + "565621826": 2.0, + "585904130": 0.23764699200789133, + "626486813": 0.6124684102833271, + "636680137": 2.0, + "706479600": 2.0, + "750448296": 0.23580759018659592, + "75070641": 2.0, + "767747424": 0.18863565661013126, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.6062410234109215, + "low": 1.1855228276572365, + "mean": 1.3999846732054932, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "outcome_ptr_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5868794326241134, + "low": 0.3124113475177305, + "mean": 0.4631205673758865, + "n": 47, + "num_samples": 500 + }, + "mean": 0.4631205673758865, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.3333333333333333, + "1475440973": 0.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.6, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.3333333333333333, + "626486813": 0.75, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.5, + "75070641": 1.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.555897435897436, + "low": 0.18666666666666668, + "mean": 0.3744494720965309, + "n": 5, + "num_samples": 500 + }, + "mean": 0.3744494720965309, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 0.6153846153846154, + "StackCube-v1": 0.6 + } + }, + "metric": "outcome_ptr_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5507246376811594, + "low": 0.3188405797101449, + "mean": 0.43478260869565216, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.1773022288327493, + "low": 0.11479435257646622, + "mean": 0.14593359597440297, + "n": 47, + "num_samples": 500 + }, + "mean": 0.14593359597440297, + "per_seed": { + "1025740212": 0.08963387331564551, + "1055157354": 0.2973966626262834, + "1055567618": 0.07486610854656817, + "1113268940": 0.1616344352768096, + "1113371261": 0.3179396489681421, + "1176943004": 0.2176350935010736, + "1189392917": 0.1628408162009477, + "1231780997": 0.18322944379343714, + "1271718007": 0.07543261691159965, + "1273247072": 0.15040783499409596, + "1358421325": 0.13502516765102715, + "1375492786": 0.19928218350064877, + "141984615": 0.10926310602682258, + "1475440973": 0.09930428958964055, + "1483645336": 0.0, + "1522272686": 0.08257819446483214, + "1528740417": 0.015990866806051107, + "1549324805": 0.19382954770507574, + "1607773860": 0.08588083120555855, + "1665739511": 0.050940808914706356, + "1747178030": 0.15825371494561297, + "1881223807": 0.019060791998604134, + "1919198188": 0.15779226512372985, + "1991414747": 0.05287821364799471, + "1997090716": 0.4044963838715546, + "2015956957": 0.18664994005668376, + "2054899126": 0.0, + "211245842": 0.16771630653845432, + "2141802816": 0.05242798758366053, + "256770874": 0.030709639081801327, + "278008524": 0.11925245416072927, + "416183106": 0.10129212810935397, + "534717141": 0.6159370955924921, + "551504635": 0.10932818664818449, + "565369392": 0.22308845281795123, + "565621826": 0.0, + "585904130": 0.10391809443539522, + "626486813": 0.16162412565990822, + "636680137": 0.1699944666253151, + "706479600": 0.0922387871049479, + "750448296": 0.14360585133877676, + "75070641": 0.2497251397544613, + "767747424": 0.21088094062197144, + "77147264": 0.06512707818376468, + "794181626": 0.12119924562800825, + "957101489": 0.20250855048512742, + "998761162": 0.23606164078349207 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.15940202325243455, + "low": 0.1233204922135519, + "mean": 0.13967424044429286, + "n": 5, + "num_samples": 500 + }, + "mean": 0.13967424044429286, + "per_task": { + "LiftPegUpright-v1": 0.1098535159049614, + "PickCube-v1": 0.15040783499409596, + "PullCube-v1": 0.17256325415222457, + "PushCube-v1": 0.13663407835877941, + "StackCube-v1": 0.128912518811403 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.17644750878364412, + "low": 0.12609473303549118, + "mean": 0.15053058510774892, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5698581560283689, + "low": 0.29964539007092195, + "mean": 0.4404255319148936, + "n": 47, + "num_samples": 500 + }, + "mean": 0.4404255319148936, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 1.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5408748114630468, + "low": 0.14666666666666667, + "mean": 0.34395173453996986, + "n": 5, + "num_samples": 500 + }, + "mean": 0.34395173453996986, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.6923076923076923, + "StackCube-v1": 0.4 + } + }, + "metric": "proposal_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5217391304347826, + "low": 0.2898550724637681, + "mean": 0.391304347826087, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.27446808510638293, + "low": -0.010638297872340425, + "mean": 0.13191489361702127, + "n": 47, + "num_samples": 500 + }, + "mean": 0.13191489361702127, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": -1.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.27076923076923076, + "low": 0.0, + "mean": 0.11692307692307694, + "n": 5, + "num_samples": 500 + }, + "mean": 0.11692307692307694, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.2 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.21739130434782608, + "low": 0.014492753623188406, + "mean": 0.11594202898550725, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2912854919257963, + "low": 0.8075891729536738, + "mean": 1.0507658735934828, + "n": 47, + "num_samples": 500 + }, + "mean": 1.0507658735934828, + "per_seed": { + "1025740212": 0.4761262834072113, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 0.6646121144294739, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.7187411189079285, + "1271718007": 2.0, + "1273247072": 0.13278456032276154, + "1358421325": 0.8142459392547607, + "1375492786": 0.0016440550098195672, + "141984615": 0.0797463705142339, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.2435554713010788, + "1528740417": 2.0, + "1549324805": 1.3427665165315072, + "1607773860": 2.0, + "1665739511": 0.8216220736503601, + "1747178030": 2.0, + "1881223807": 0.0513521283864975, + "1919198188": 0.08992795646190643, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.45124312303960323, + "2054899126": 2.0, + "211245842": 0.8130142092704773, + "2141802816": 0.12472955882549286, + "256770874": 0.24107472598552704, + "278008524": 0.011432985425926745, + "416183106": 0.5778142213821411, + "534717141": 2.0, + "551504635": 0.7250897884368896, + "565369392": 0.7852714310089747, + "565621826": 2.0, + "585904130": 0.05598785417775313, + "626486813": 0.5615248780231923, + "636680137": 0.012414653785526752, + "706479600": 2.0, + "750448296": 0.1069931760430336, + "75070641": 0.2396746575832367, + "767747424": 0.5703969299793243, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 0.5881698727607727, + "998761162": 0.08403940498828888 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.2200616387067698, + "low": 0.4891751624122831, + "mean": 0.8746016368245654, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8746016368245654, + "per_task": { + "LiftPegUpright-v1": 1.0974282175302505, + "PickCube-v1": 0.13278456032276154, + "PullCube-v1": 0.6413554182680159, + "PushCube-v1": 1.406166712825115, + "StackCube-v1": 1.095273275176684 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.149046370209586, + "low": 0.73185485545917, + "mean": 0.9164160069709212, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5003876065769306, + "low": -0.0012803047515636864, + "mean": 0.23716175138788523, + "n": 47, + "num_samples": 500 + }, + "mean": 0.23716175138788523, + "per_seed": { + "1025740212": -0.05224737524986267, + "1055157354": 1.8235089033842087, + "1055567618": 0.0, + "1113268940": 0.24229958653450012, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": -0.008265018463134766, + "1271718007": 0.5857395231723785, + "1273247072": -0.02064509689807892, + "1358421325": -0.018456339836120605, + "1375492786": -1.9983559449901804, + "141984615": -0.01406576856970787, + "1475440973": 0.0, + "1483645336": 1.9835839420557022, + "1522272686": 0.048794373869895935, + "1528740417": 0.0, + "1549324805": -0.035980931793649994, + "1607773860": 0.0, + "1665739511": -1.17837792634964, + "1747178030": 1.8536934107542038, + "1881223807": 0.0193251371383667, + "1919198188": 0.05272909998893738, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": 0.38412333317101, + "2054899126": 0.0, + "211245842": -0.02582019567489624, + "2141802816": 0.05392231047153473, + "256770874": -0.6146733313798904, + "278008524": -0.09603888296987861, + "416183106": 0.37748780846595764, + "534717141": 0.0, + "551504635": 0.0011172294616699219, + "565369392": -0.014494215448697409, + "565621826": 0.0, + "585904130": -0.019304243847727776, + "626486813": 0.0560848613968119, + "636680137": -1.9875853462144732, + "706479600": 1.1874593496322632, + "750448296": -0.004540383815765381, + "75070641": 0.03692302107810974, + "767747424": 0.022872742265462875, + "77147264": 0.5784205198287964, + "794181626": 1.826662853360176, + "957101489": 0.3842049241065979, + "998761162": 0.015134349465370178 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4710757725513899, + "low": -0.00611102568926107, + "mean": 0.20618344864719879, + "n": 5, + "num_samples": 500 + }, + "mean": 0.20618344864719879, + "per_task": { + "LiftPegUpright-v1": 0.030484070380528767, + "PickCube-v1": -0.02064509689807892, + "PullCube-v1": -0.009874502515338142, + "PushCube-v1": 0.7374794540496973, + "StackCube-v1": 0.2934733182191849 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.37626325682588463, + "low": 0.0164921339938714, + "mean": 0.2002298393593831, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4219858156028369, + "low": 0.17978723404255317, + "mean": 0.2914893617021277, + "n": 47, + "num_samples": 500 + }, + "mean": 0.2914893617021277, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 1.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.40307692307692305, + "low": 0.0796078431372549, + "mean": 0.24063348416289593, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24063348416289593, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.2647058823529412, + "PushCube-v1": 0.5384615384615384, + "StackCube-v1": 0.06666666666666667 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.37681159420289856, + "low": 0.18840579710144928, + "mean": 0.2753623188405797, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.12127659574468086, + "low": -0.16595744680851063, + "mean": -0.01702127659574468, + "n": 47, + "num_samples": 500 + }, + "mean": -0.01702127659574468, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": -1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": -1.0, + "1665739511": -1.0, + "1747178030": 1.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": -1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": -1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": -0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.12669683257918551, + "low": -0.08588235294117648, + "mean": 0.013604826546003018, + "n": 5, + "num_samples": 500 + }, + "mean": 0.013604826546003018, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.029411764705882353, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": -0.13333333333333333 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.11594202898550725, + "low": -0.10144927536231885, + "mean": 0.0, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9708039672328622, + "low": 0.5131759628765814, + "mean": 0.729357396943597, + "n": 47, + "num_samples": 500 + }, + "mean": 0.729357396943597, + "per_seed": { + "1025740212": 0.21424125134944916, + "1055157354": 0.14835423231124878, + "1055567618": 0.8168438673019409, + "1113268940": 0.27990618348121643, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.22899846732616425, + "1271718007": 2.0, + "1273247072": 0.13278456032276154, + "1358421325": 0.23671336472034454, + "1375492786": 0.001358628273010254, + "141984615": 0.04657847310105959, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.2340310961008072, + "1528740417": 2.0, + "1549324805": 1.3393354682872693, + "1607773860": 0.220267653465271, + "1665739511": 0.8140708208084106, + "1747178030": 2.0, + "1881223807": 0.03111603856086731, + "1919198188": 0.07307347655296326, + "1991414747": 2.0, + "1997090716": 0.03083018958568573, + "2015956957": 0.433539473824203, + "2054899126": 2.0, + "211245842": 0.22217948734760284, + "2141802816": 0.0492904931306839, + "256770874": 0.13286492228507996, + "278008524": 0.009787360846530646, + "416183106": 0.24176457524299622, + "534717141": 0.07250408083200455, + "551504635": 0.6918362975120544, + "565369392": 0.7208344464500746, + "565621826": 2.0, + "585904130": 0.012818952556699514, + "626486813": 0.5429544361541048, + "636680137": 0.012414653785526752, + "706479600": 0.8134256601333618, + "750448296": 0.029670913703739643, + "75070641": 0.16485285758972168, + "767747424": 0.554436425678432, + "77147264": 0.13140545785427094, + "794181626": 2.0, + "957101489": 0.5881698727607727, + "998761162": 0.006543517112731934 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9089970031332648, + "low": 0.35030357998435147, + "mean": 0.629650291558808, + "n": 5, + "num_samples": 500 + }, + "mean": 0.629650291558808, + "per_task": { + "LiftPegUpright-v1": 0.9024970332781473, + "PickCube-v1": 0.13278456032276154, + "PullCube-v1": 0.560690138827982, + "PushCube-v1": 1.0896504051410234, + "StackCube-v1": 0.4626293202241262 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.8693176263022115, + "low": 0.48426623860259366, + "mean": 0.6625525587005541, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.19248003160666863, + "low": -0.359282114742413, + "mean": -0.0842467252620007, + "n": 47, + "num_samples": 500 + }, + "mean": -0.0842467252620007, + "per_seed": { + "1025740212": -0.3141324073076248, + "1055157354": -0.02813686430454254, + "1055567618": -1.183156132698059, + "1113268940": -0.14240634441375732, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": -0.498007670044899, + "1271718007": 0.5857395231723785, + "1273247072": -0.02064509689807892, + "1358421325": -0.5959889143705368, + "1375492786": -1.9986413717269897, + "141984615": -0.04723366598288218, + "1475440973": 0.0, + "1483645336": 1.9835839420557022, + "1522272686": 0.03926999866962433, + "1528740417": 0.0, + "1549324805": -0.03941198003788789, + "1607773860": -1.779732346534729, + "1665739511": -1.1859291791915894, + "1747178030": 1.8536934107542038, + "1881223807": -0.0009109526872634888, + "1919198188": 0.0358746200799942, + "1991414747": 0.0, + "1997090716": -0.10844460129737854, + "2015956957": 0.3664196839556098, + "2054899126": 0.0, + "211245842": -0.6166549175977707, + "2141802816": -0.02151675522327423, + "256770874": -0.7228831350803375, + "278008524": -0.09768450754927471, + "416183106": 0.041438162326812744, + "534717141": -1.9274959191679955, + "551504635": -0.03213626146316528, + "565369392": -0.07893120000759761, + "565621826": 0.0, + "585904130": -0.06247314546878139, + "626486813": 0.037514419527724385, + "636680137": -1.9875853462144732, + "706479600": 0.000885009765625, + "750448296": -0.08186264615505934, + "75070641": -0.03789877891540527, + "767747424": 0.006912237964570522, + "77147264": -1.2901740223169327, + "794181626": 1.826662853360176, + "957101489": 0.3842049241065979, + "998761162": -0.06236153841018677 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.20158050865397423, + "low": -0.25449976120541307, + "mean": -0.038767896618558506, + "n": 5, + "num_samples": 500 + }, + "mean": -0.038767896618558506, + "per_task": { + "LiftPegUpright-v1": -0.1644471138715744, + "PickCube-v1": -0.02064509689807892, + "PullCube-v1": -0.0905397819553721, + "PushCube-v1": 0.4209631463656059, + "StackCube-v1": -0.339170636733373 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.14216782323901367, + "low": -0.2450303274233812, + "mean": -0.053633608910983996, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5197332343231088, + "low": 0.16253579634643023, + "mean": 0.321408476649886, + "n": 47, + "num_samples": 500 + }, + "mean": 0.321408476649886, + "per_seed": { + "1025740212": 0.26188503205776215, + "1055157354": 1.8516457676887512, + "1055567618": 1.183156132698059, + "1113268940": 0.38470593094825745, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.4897426515817642, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.5775325745344162, + "1375492786": 0.0002854267368093133, + "141984615": 0.03316789741317431, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.009524375200271606, + "1528740417": 0.0, + "1549324805": 0.0034310482442378998, + "1607773860": 1.779732346534729, + "1665739511": 0.007551252841949463, + "1747178030": 0.0, + "1881223807": 0.020236089825630188, + "1919198188": 0.016854479908943176, + "1991414747": 0.0, + "1997090716": 1.9691698104143143, + "2015956957": 0.01770364921540022, + "2054899126": 0.0, + "211245842": 0.5908347219228745, + "2141802816": 0.07543906569480896, + "256770874": 0.10820980370044708, + "278008524": 0.0016456245793960989, + "416183106": 0.3360496461391449, + "534717141": 1.9274959191679955, + "551504635": 0.033253490924835205, + "565369392": 0.06443698455890019, + "565621826": 0.0, + "585904130": 0.043168901621053614, + "626486813": 0.018570441869087517, + "636680137": 0.0, + "706479600": 1.1865743398666382, + "750448296": 0.07732226233929396, + "75070641": 0.07482179999351501, + "767747424": 0.015960504300892353, + "77147264": 1.868594542145729, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.07749588787555695 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5061151639620463, + "low": 0.07125234862643422, + "mean": 0.24495134526575732, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24495134526575732, + "per_task": { + "LiftPegUpright-v1": 0.19493118425210318, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08066527944003396, + "PushCube-v1": 0.31651630768409145, + "StackCube-v1": 0.6326439549525579 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.378890726060701, + "low": 0.13165242864372398, + "mean": 0.2538634482703671, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2553191489361702, + "low": 0.0425531914893617, + "mean": 0.14893617021276595, + "n": 47, + "num_samples": 500 + }, + "mean": 0.14893617021276595, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2615384615384615, + "low": 0.011764705882352941, + "mean": 0.1033182503770739, + "n": 5, + "num_samples": 500 + }, + "mean": 0.1033182503770739, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.029411764705882353, + "PushCube-v1": 0.15384615384615385, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.18840579710144928, + "low": 0.043478260869565216, + "mean": 0.11594202898550725, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5531914893617021, + "low": 0.2695035460992908, + "mean": 0.40070921985815605, + "n": 47, + "num_samples": 500 + }, + "mean": 0.40070921985815605, + "per_seed": { + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7948717948717949, + "low": 0.21683257918552035, + "mean": 0.5266365007541478, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5266365007541478, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.058823529411764705, + "PushCube-v1": 0.3076923076923077, + "StackCube-v1": 0.6 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.391304347826087, + "low": 0.17391304347826086, + "mean": 0.2898550724637681, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6950354609929079, + "low": 0.4113475177304965, + "mean": 0.549645390070922, + "n": 47, + "num_samples": 500 + }, + "mean": 0.549645390070922, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9066666666666666, + "low": 0.3122171945701358, + "mean": 0.6299547511312216, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6299547511312216, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.08823529411764706, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.9333333333333333 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5217391304347826, + "low": 0.2753623188405797, + "mean": 0.4057971014492754, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8930026109046337, + "low": 0.45526805628827577, + "mean": 0.6654373238655799, + "n": 47, + "num_samples": 500 + }, + "mean": 0.6654373238655799, + "per_seed": { + "1025740212": 1.5238737165927887, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 1.3353878855705261, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.2812588810920715, + "1271718007": 0.0, + "1273247072": 1.8672154396772385, + "1358421325": 1.1857540607452393, + "1375492786": 0.24180691444780678, + "141984615": 0.12070147196451823, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.7564445286989212, + "1528740417": 0.0, + "1549324805": 0.05998400536676248, + "1607773860": 0.0, + "1665739511": 1.17837792634964, + "1747178030": 0.0, + "1881223807": 1.9486478716135025, + "1919198188": 1.9100720435380936, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.07863824404776096, + "2054899126": 0.0, + "211245842": 1.1869857907295227, + "2141802816": 1.8752704411745071, + "256770874": 1.758925274014473, + "278008524": 1.1023782094707713, + "416183106": 1.4221857786178589, + "534717141": 0.0, + "551504635": 1.2749102115631104, + "565369392": 0.6413601487874985, + "565621826": 0.0, + "585904130": 0.1816591378301382, + "626486813": 0.0801479306537658, + "636680137": 1.9875853462144732, + "706479600": 0.0, + "750448296": 0.12881441414356232, + "75070641": 1.7603253424167633, + "767747424": 0.05905248410999775, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.4118301272392273, + "998761162": 1.9159605950117111 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.4200412662059834, + "low": 0.41746960569771857, + "mean": 0.8929219812617306, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8929219812617306, + "per_task": { + "LiftPegUpright-v1": 0.9025717824697495, + "PickCube-v1": 1.8672154396772385, + "PullCube-v1": 0.1962626721634639, + "PushCube-v1": 0.593833287174885, + "StackCube-v1": 0.904726724823316 + } + }, + "metric": "support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.6783581883714035, + "low": 0.3298995522124207, + "mean": 0.5108160955605326, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.235173605110724, + "low": 0.7539350982827253, + "mean": 0.9865234781741099, + "n": 47, + "num_samples": 500 + }, + "mean": 0.9865234781741099, + "per_seed": { + "1025740212": 1.7857587486505508, + "1055157354": 1.8516457676887512, + "1055567618": 1.183156132698059, + "1113268940": 1.7200938165187836, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.7710015326738358, + "1271718007": 0.0, + "1273247072": 1.8672154396772385, + "1358421325": 1.7632866352796555, + "1375492786": 0.2420923411846161, + "141984615": 0.15386936937769255, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.7659689038991928, + "1528740417": 0.0, + "1549324805": 0.06341505361100037, + "1607773860": 1.779732346534729, + "1665739511": 1.1859291791915894, + "1747178030": 0.0, + "1881223807": 1.9688839614391327, + "1919198188": 1.9269265234470367, + "1991414747": 0.0, + "1997090716": 1.9691698104143143, + "2015956957": 0.09634189326316119, + "2054899126": 0.0, + "211245842": 1.7778205126523972, + "2141802816": 1.950709506869316, + "256770874": 1.86713507771492, + "278008524": 1.1040238340501674, + "416183106": 1.7582354247570038, + "534717141": 1.9274959191679955, + "551504635": 1.3081637024879456, + "565369392": 0.7057971333463987, + "565621826": 0.0, + "585904130": 0.2248280394511918, + "626486813": 0.08356922247912735, + "636680137": 1.9875853462144732, + "706479600": 1.1865743398666382, + "750448296": 0.20613667648285627, + "75070641": 1.8351471424102783, + "767747424": 0.0750129884108901, + "77147264": 1.868594542145729, + "794181626": 0.0, + "957101489": 1.4118301272392273, + "998761162": 1.993456482887268 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6098733187333132, + "low": 0.6546314741212763, + "mean": 1.1375168759382237, + "n": 5, + "num_samples": 500 + }, + "mean": 1.1375168759382237, + "per_task": { + "LiftPegUpright-v1": 1.0975029667218525, + "PickCube-v1": 1.8672154396772385, + "PullCube-v1": 0.2751456986571772, + "PushCube-v1": 0.9103495948589765, + "StackCube-v1": 1.5373706797758737 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9371206429449545, + "low": 0.5655091602922134, + "mean": 0.763801332234162, + "n": 69, + "num_samples": 500 + }, + "n": 69 + } + }, + "num_rows": 69, + "report_type": "ctt_generated_measured_rollout", + "schema_version": 1, + "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957", + "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "success_summary": { + "base_success_rate": 0.2753623188405797, + "base_utility_mean": 0.7161861676115381, + "hidden_chart_oracle_success_rate": 0.6666666666666666, + "hidden_chart_oracle_utility_mean": 1.3999846732054932, + "max_restore_error": 1.4901161193847656e-08, + "proposal_oracle_success_gain_over_base": 0.11594202898550725, + "proposal_oracle_success_rate": 0.391304347826087, + "proposal_oracle_utility_mean": 0.9164160069709212, + "selected_success_gain_over_base": 0.0, + "selected_success_rate": 0.2753623188405797, + "selected_utility_mean": 0.6625525587005541, + "success_selector_gap": 0.11594202898550725, + "success_support_gap": 0.2898550724637681 + }, + "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42", + "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381" +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..c1f2e5ecf7f8d78fe849abbfd7ee49e5d29eddae --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_seed.json @@ -0,0 +1,1083 @@ +{ + "1025740212": { + "base_success": 0.0, + "base_utility": 0.528373658657074, + "branch_car_at_8": 0.26188503205776215, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.08963387331564551, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4761262834072113, + "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21424125134944916, + "selected_utility_gain_over_base_at_8": -0.3141324073076248, + "selector_regret_at_8": 0.26188503205776215, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5238737165927887, + "total_car_to_hidden_at_8": 1.7857587486505508 + }, + "1055157354": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, + "branch_car_at_8": 1.8516457676887512, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2973966626262834, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14835423231124878, + "selected_utility_gain_over_base_at_8": -0.02813686430454254, + "selector_regret_at_8": 1.8516457676887512, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8516457676887512 + }, + "1055567618": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.183156132698059, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07486610854656817, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8168438673019409, + "selected_utility_gain_over_base_at_8": -1.183156132698059, + "selector_regret_at_8": 1.183156132698059, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.183156132698059 + }, + "1113268940": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, + "branch_car_at_8": 0.38470593094825745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1616344352768096, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6646121144294739, + "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.27990618348121643, + "selected_utility_gain_over_base_at_8": -0.14240634441375732, + "selector_regret_at_8": 0.38470593094825745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3353878855705261, + "total_car_to_hidden_at_8": 1.7200938165187836 + }, + "1113371261": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.3179396489681421, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1176943004": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2176350935010736, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1231780997": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, + "branch_car_at_8": 0.4897426515817642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.18322944379343714, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7187411189079285, + "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22899846732616425, + "selected_utility_gain_over_base_at_8": -0.498007670044899, + "selector_regret_at_8": 0.4897426515817642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2812588810920715, + "total_car_to_hidden_at_8": 1.7710015326738358 + }, + "1271718007": { + "base_success": 0.5, + "base_utility": 1.4142604768276215, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.07543261691159965, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5857395231723785, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5857395231723785, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 + }, + "1358421325": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, + "branch_car_at_8": 0.5775325745344162, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.13502516765102715, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8142459392547607, + "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23671336472034454, + "selected_utility_gain_over_base_at_8": -0.5959889143705368, + "selector_regret_at_8": 0.5775325745344162, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1857540607452393, + "total_car_to_hidden_at_8": 1.7632866352796555 + }, + "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0002854267368093133, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, + "selector_regret_at_8": 0.0002854267368093133, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24180691444780678, + "total_car_to_hidden_at_8": 0.2420923411846161 + }, + "141984615": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, + "branch_car_at_8": 0.03316789741317431, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.10926310602682258, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0797463705142339, + "proposal_oracle_utility_gain_over_base_at_8": -0.01406576856970787, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04657847310105959, + "selected_utility_gain_over_base_at_8": -0.04723366598288218, + "selector_regret_at_8": 0.03316789741317431, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12070147196451823, + "total_car_to_hidden_at_8": 0.15386936937769255 + }, + "1475440973": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.09930428958964055, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1483645336": { + "base_success": 0.0, + "base_utility": 0.01641605794429779, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.9835839420557022, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1522272686": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, + "branch_car_at_8": 0.009524375200271606, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.08257819446483214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2435554713010788, + "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2340310961008072, + "selected_utility_gain_over_base_at_8": 0.03926999866962433, + "selector_regret_at_8": 0.009524375200271606, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7564445286989212, + "total_car_to_hidden_at_8": 1.7659689038991928 + }, + "1528740417": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.015990866806051107, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1549324805": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, + "branch_car_at_8": 0.0034310482442378998, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19382954770507574, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.3427665165315072, + "proposal_oracle_utility_gain_over_base_at_8": -0.035980931793649994, + "selected_success_at_8": 0.6666666666666666, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.3393354682872693, + "selected_utility_gain_over_base_at_8": -0.03941198003788789, + "selector_regret_at_8": 0.0034310482442378998, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05998400536676248, + "total_car_to_hidden_at_8": 0.06341505361100037 + }, + "1607773860": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.779732346534729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.08588083120555855, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.220267653465271, + "selected_utility_gain_over_base_at_8": -1.779732346534729, + "selector_regret_at_8": 1.779732346534729, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.779732346534729 + }, + "1665739511": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.007551252841949463, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.050940808914706356, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8216220736503601, + "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8140708208084106, + "selected_utility_gain_over_base_at_8": -1.1859291791915894, + "selector_regret_at_8": 0.007551252841949463, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.17837792634964, + "total_car_to_hidden_at_8": 1.1859291791915894 + }, + "1747178030": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.15825371494561297, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8536934107542038, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.020236089825630188, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, + "selector_regret_at_8": 0.020236089825630188, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9486478716135025, + "total_car_to_hidden_at_8": 1.9688839614391327 + }, + "1919198188": { + "base_success": 0.0, + "base_utility": 0.037198856472969055, + "branch_car_at_8": 0.016854479908943176, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.15779226512372985, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08992795646190643, + "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07307347655296326, + "selected_utility_gain_over_base_at_8": 0.0358746200799942, + "selector_regret_at_8": 0.016854479908943176, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9100720435380936, + "total_car_to_hidden_at_8": 1.9269265234470367 + }, + "1991414747": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.05287821364799471, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 1.9691698104143143, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, + "selector_regret_at_8": 1.9691698104143143, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9691698104143143 + }, + "2015956957": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, + "branch_car_at_8": 0.01770364921540022, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.18664994005668376, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.45124312303960323, + "proposal_oracle_utility_gain_over_base_at_8": 0.38412333317101, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.433539473824203, + "selected_utility_gain_over_base_at_8": 0.3664196839556098, + "selector_regret_at_8": 0.01770364921540022, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07863824404776096, + "total_car_to_hidden_at_8": 0.09634189326316119 + }, + "2054899126": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "211245842": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, + "branch_car_at_8": 0.5908347219228745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.16771630653845432, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8130142092704773, + "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22217948734760284, + "selected_utility_gain_over_base_at_8": -0.6166549175977707, + "selector_regret_at_8": 0.5908347219228745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1869857907295227, + "total_car_to_hidden_at_8": 1.7778205126523972 + }, + "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 0.07543906569480896, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, + "selector_regret_at_8": 0.07543906569480896, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8752704411745071, + "total_car_to_hidden_at_8": 1.950709506869316 + }, + "256770874": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, + "branch_car_at_8": 0.10820980370044708, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.030709639081801327, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.24107472598552704, + "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13286492228507996, + "selected_utility_gain_over_base_at_8": -0.7228831350803375, + "selector_regret_at_8": 0.10820980370044708, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.758925274014473, + "total_car_to_hidden_at_8": 1.86713507771492 + }, + "278008524": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, + "branch_car_at_8": 0.0016456245793960989, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.11925245416072927, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.011432985425926745, + "proposal_oracle_utility_gain_over_base_at_8": -0.09603888296987861, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.009787360846530646, + "selected_utility_gain_over_base_at_8": -0.09768450754927471, + "selector_regret_at_8": 0.0016456245793960989, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.1023782094707713, + "total_car_to_hidden_at_8": 1.1040238340501674 + }, + "416183106": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, + "branch_car_at_8": 0.3360496461391449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10129212810935397, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5778142213821411, + "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.24176457524299622, + "selected_utility_gain_over_base_at_8": 0.041438162326812744, + "selector_regret_at_8": 0.3360496461391449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4221857786178589, + "total_car_to_hidden_at_8": 1.7582354247570038 + }, + "534717141": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9274959191679955, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.6159370955924921, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07250408083200455, + "selected_utility_gain_over_base_at_8": -1.9274959191679955, + "selector_regret_at_8": 1.9274959191679955, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9274959191679955 + }, + "551504635": { + "base_success": 0.0, + "base_utility": 0.7239725589752197, + "branch_car_at_8": 0.033253490924835205, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10932818664818449, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7250897884368896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6918362975120544, + "selected_utility_gain_over_base_at_8": -0.03213626146316528, + "selector_regret_at_8": 0.033253490924835205, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2749102115631104, + "total_car_to_hidden_at_8": 1.3081637024879456 + }, + "565369392": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, + "branch_car_at_8": 0.06443698455890019, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.22308845281795123, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7852714310089747, + "proposal_oracle_utility_gain_over_base_at_8": -0.014494215448697409, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7208344464500746, + "selected_utility_gain_over_base_at_8": -0.07893120000759761, + "selector_regret_at_8": 0.06443698455890019, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.6413601487874985, + "total_car_to_hidden_at_8": 0.7057971333463987 + }, + "565621826": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "585904130": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, + "branch_car_at_8": 0.043168901621053614, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.10391809443539522, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05598785417775313, + "proposal_oracle_utility_gain_over_base_at_8": -0.019304243847727776, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012818952556699514, + "selected_utility_gain_over_base_at_8": -0.06247314546878139, + "selector_regret_at_8": 0.043168901621053614, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1816591378301382, + "total_car_to_hidden_at_8": 0.2248280394511918 + }, + "626486813": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, + "branch_car_at_8": 0.018570441869087517, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.16162412565990822, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5615248780231923, + "proposal_oracle_utility_gain_over_base_at_8": 0.0560848613968119, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5429544361541048, + "selected_utility_gain_over_base_at_8": 0.037514419527724385, + "selector_regret_at_8": 0.018570441869087517, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0801479306537658, + "total_car_to_hidden_at_8": 0.08356922247912735 + }, + "636680137": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.1699944666253151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012414653785526752, + "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.012414653785526752, + "selected_utility_gain_over_base_at_8": -1.9875853462144732, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9875853462144732, + "total_car_to_hidden_at_8": 1.9875853462144732 + }, + "706479600": { + "base_success": 0.0, + "base_utility": 0.8125406503677368, + "branch_car_at_8": 1.1865743398666382, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0922387871049479, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8134256601333618, + "selected_utility_gain_over_base_at_8": 0.000885009765625, + "selector_regret_at_8": 1.1865743398666382, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.1865743398666382 + }, + "750448296": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, + "branch_car_at_8": 0.07732226233929396, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.14360585133877676, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1069931760430336, + "proposal_oracle_utility_gain_over_base_at_8": -0.004540383815765381, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.029670913703739643, + "selected_utility_gain_over_base_at_8": -0.08186264615505934, + "selector_regret_at_8": 0.07732226233929396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12881441414356232, + "total_car_to_hidden_at_8": 0.20613667648285627 + }, + "75070641": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, + "branch_car_at_8": 0.07482179999351501, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2497251397544613, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2396746575832367, + "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16485285758972168, + "selected_utility_gain_over_base_at_8": -0.03789877891540527, + "selector_regret_at_8": 0.07482179999351501, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7603253424167633, + "total_car_to_hidden_at_8": 1.8351471424102783 + }, + "767747424": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, + "branch_car_at_8": 0.015960504300892353, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, + "outcome_ptr_at_8": 0.25, + "pairwise_causal_calibration_ece": 0.21088094062197144, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5703969299793243, + "proposal_oracle_utility_gain_over_base_at_8": 0.022872742265462875, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.554436425678432, + "selected_utility_gain_over_base_at_8": 0.006912237964570522, + "selector_regret_at_8": 0.015960504300892353, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05905248410999775, + "total_car_to_hidden_at_8": 0.0750129884108901 + }, + "77147264": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, + "branch_car_at_8": 1.868594542145729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.06512707818376468, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.5, + "selected_utility_at_8": 0.13140545785427094, + "selected_utility_gain_over_base_at_8": -1.2901740223169327, + "selector_regret_at_8": 1.868594542145729, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.868594542145729 + }, + "794181626": { + "base_success": 0.0, + "base_utility": 0.1733371466398239, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.12119924562800825, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.826662853360176, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "957101489": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.20250855048512742, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5881698727607727, + "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5881698727607727, + "selected_utility_gain_over_base_at_8": 0.3842049241065979, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4118301272392273, + "total_car_to_hidden_at_8": 1.4118301272392273 + }, + "998761162": { + "base_success": 0.0, + "base_utility": 0.0689050555229187, + "branch_car_at_8": 0.07749588787555695, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.23606164078349207, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08403940498828888, + "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006543517112731934, + "selected_utility_gain_over_base_at_8": -0.06236153841018677, + "selector_regret_at_8": 0.07749588787555695, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9159605950117111, + "total_car_to_hidden_at_8": 1.993456482887268 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..221883cb828eead25cb9dc4e96dca084a508ea33 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.0669441471497219, + "branch_car_at_8": 0.19493118425210318, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.1098535159049614, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.0974282175302505, + "proposal_oracle_utility_gain_over_base_at_8": 0.030484070380528767, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.9024970332781473, + "selected_utility_gain_over_base_at_8": -0.1644471138715744, + "selector_regret_at_8": 0.19493118425210318, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.9025717824697495, + "total_car_to_hidden_at_8": 1.0975029667218525 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 + }, + "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, + "branch_car_at_8": 0.08066527944003396, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, + "outcome_ptr_at_8": 0.3235294117647059, + "pairwise_causal_calibration_ece": 0.17256325415222457, + "proposal_oracle_success_at_8": 0.29411764705882354, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6413554182680159, + "proposal_oracle_utility_gain_over_base_at_8": -0.009874502515338142, + "selected_success_at_8": 0.2647058823529412, + "selected_success_gain_over_base_at_8": -0.029411764705882353, + "selected_utility_at_8": 0.560690138827982, + "selected_utility_gain_over_base_at_8": -0.0905397819553721, + "selector_regret_at_8": 0.08066527944003396, + "success_selector_gap_at_8": 0.029411764705882353, + "success_support_gap_at_8": 0.058823529411764705, + "success_total_car_to_hidden_at_8": 0.08823529411764706, + "support_gap_at_8": 0.1962626721634639, + "total_car_to_hidden_at_8": 0.2751456986571772 + }, + "PushCube-v1": { + "base_success": 0.3076923076923077, + "base_utility": 0.6686872587754176, + "branch_car_at_8": 0.31651630768409145, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6153846153846154, + "pairwise_causal_calibration_ece": 0.13663407835877941, + "proposal_oracle_success_at_8": 0.6923076923076923, + "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464, + "proposal_oracle_utility_at_8": 1.406166712825115, + "proposal_oracle_utility_gain_over_base_at_8": 0.7374794540496973, + "selected_success_at_8": 0.5384615384615384, + "selected_success_gain_over_base_at_8": 0.23076923076923078, + "selected_utility_at_8": 1.0896504051410234, + "selected_utility_gain_over_base_at_8": 0.4209631463656059, + "selector_regret_at_8": 0.31651630768409145, + "success_selector_gap_at_8": 0.15384615384615385, + "success_support_gap_at_8": 0.3076923076923077, + "success_total_car_to_hidden_at_8": 0.46153846153846156, + "support_gap_at_8": 0.593833287174885, + "total_car_to_hidden_at_8": 0.9103495948589765 + }, + "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8017999569574992, + "branch_car_at_8": 0.6326439549525579, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.128912518811403, + "proposal_oracle_success_at_8": 0.4, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 1.095273275176684, + "proposal_oracle_utility_gain_over_base_at_8": 0.2934733182191849, + "selected_success_at_8": 0.06666666666666667, + "selected_success_gain_over_base_at_8": -0.13333333333333333, + "selected_utility_at_8": 0.4626293202241262, + "selected_utility_gain_over_base_at_8": -0.339170636733373, + "selector_regret_at_8": 0.6326439549525579, + "success_selector_gap_at_8": 0.3333333333333333, + "success_support_gap_at_8": 0.6, + "success_total_car_to_hidden_at_8": 0.9333333333333333, + "support_gap_at_8": 0.904726724823316, + "total_car_to_hidden_at_8": 1.5373706797758737 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/source_data_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed0/source_data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6cc02ca1808e0515839939b88beab0acaad5cbc --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/source_data_hash.txt @@ -0,0 +1 @@ +90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/source_split_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed0/source_split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..26988e766c6751325c7f72d33b199bd133350cde --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/source_split_hash.txt @@ -0,0 +1 @@ +867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/split_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed0/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..8533e80f235606cef4ba0119a3ac3806d63a0983 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/split_hash.txt @@ -0,0 +1 @@ +9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/table.tex b/workspace/runs/ctt_residual_rollout_val69_seed0/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..8ef54f8e7b0c023991b43b48494640a2c662ca34 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/table.tex @@ -0,0 +1,28 @@ +% Auto-generated by scripts/eval_metrics.py +\begin{tabular}{lrrrr} +\toprule +Metric & N & Micro mean & CI low & CI high \\ +\midrule +base\_success & 69 & 0.2754 & 0.1739 & 0.3913 \\ +base\_utility & 69 & 0.7162 & 0.5461 & 0.9125 \\ +branch\_car\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\ +hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\ +outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3188 & 0.5507 \\ +pairwise\_causal\_calibration\_ece & 69 & 0.1505 & 0.1261 & 0.1764 \\ +proposal\_oracle\_success\_at\_8 & 69 & 0.3913 & 0.2899 & 0.5217 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1159 & 0.0145 & 0.2174 \\ +proposal\_oracle\_utility\_at\_8 & 69 & 0.9164 & 0.7319 & 1.1490 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.2002 & 0.0165 & 0.3763 \\ +selected\_success\_at\_8 & 69 & 0.2754 & 0.1884 & 0.3768 \\ +selected\_success\_gain\_over\_base\_at\_8 & 69 & 0.0000 & -0.1014 & 0.1159 \\ +selected\_utility\_at\_8 & 69 & 0.6626 & 0.4843 & 0.8693 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.0536 & -0.2450 & 0.1422 \\ +selector\_regret\_at\_8 & 69 & 0.2539 & 0.1317 & 0.3789 \\ +success\_selector\_gap\_at\_8 & 69 & 0.1159 & 0.0435 & 0.1884 \\ +success\_support\_gap\_at\_8 & 69 & 0.2899 & 0.1739 & 0.3913 \\ +success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\ +support\_gap\_at\_8 & 69 & 0.5108 & 0.3299 & 0.6784 \\ +total\_car\_to\_hidden\_at\_8 & 69 & 0.7638 & 0.5655 & 0.9371 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed0/train.log b/workspace/runs/ctt_residual_rollout_val69_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..921e09ec7e894ba5250fa0a627691cdfb6e25e63 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed0/train.log @@ -0,0 +1 @@ +CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed0/model.pt diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/data_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed1/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..a201880dd0bef2025b061d137204bb9db2f137c2 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/data_hash.txt @@ -0,0 +1 @@ +fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/eval.log b/workspace/runs/ctt_residual_rollout_val69_seed1/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..fa69ab9f3c5b7f13d101266153ca010835e5a21a --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/eval.log @@ -0,0 +1,668 @@ +start +importing gymnasium/mani_skill +imported gymnasium/mani_skill +loaded checkpoint=runs/ctt_residual_full_seed1/model.pt +loaded charts source=301 target=69 +generated proposals rows=69 total=552 +env init sequential task=LiftPegUpright-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=LiftPegUpright-v1 +env init sequential task=LiftPegUpright-v1 batch=2 sim=physx_cpu render=cpu +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=LiftPegUpright-v1 +env init sequential task=PickCube-v1 batch=1 sim=physx_cpu render=cpu +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PickCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=2 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=1 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=3 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +rollout complete rows=69 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics.json b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics.json index e9a46a147419efd486bd4b6b16f9e648c98e65f2..62afceb2e4bcafedbca716e8d7b5bc3276da98e0 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics.json @@ -5,766 +5,2050 @@ "num_rows": 69, "rows": [ { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.0, "chart_id": "ms-484ea653bbca2de3a4a2", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28322114525972475, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368, "seed": "1231780997", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.2729938626289368, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.47739143669605255, "chart_id": "ms-4b4cd12d70c0615e0930", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12992986623548633, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6983401775360107, + "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677, "seed": "1025740212", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2209487408399582, + "selected_utility_gain_over_base_at_8": -0.3074249178171158, "selector_regret_at_8": 0.47739143669605255, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.3016598224639893, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7790512591600418 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.289921224117279, "chart_id": "ms-6e268d1e2915740816a1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2443552809335613, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1475440973", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.710078775882721, + "selected_utility_gain_over_base_at_8": -1.289921224117279, "selector_regret_at_8": 1.289921224117279, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.289921224117279 }, { + "base_success": 0.0, + "base_utility": 0.7247228622436523, "branch_car_at_8": 0.0, "chart_id": "ms-8864796e31bbe93d7c5d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2299325778712667, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7224790453910828, + "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958, "seed": "551504635", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7224790453910828, + "selected_utility_gain_over_base_at_8": -0.00224381685256958, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2775209546089172, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.2775209546089172 }, { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.18306860327720642, "chart_id": "ms-b04b489f4fbad93b65ca", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10725283874677279, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6617497801780701, + "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963, "seed": "1113268940", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.47868117690086365, + "selected_utility_gain_over_base_at_8": 0.05636864900588989, "selector_regret_at_8": 0.18306860327720642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.33825021982193, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.5213188230991364 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c663f7e73e7e8376a05c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2757671830981365, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1528740417", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.45726464316248894, "chart_id": "ms-2458fcfffb3f96c6ca1a", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, "seed": "1273247072", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, "selector_regret_at_8": 0.45726464316248894, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.500250667333603, - "task_id": "PickCube-v1" + "task_id": "PickCube-v1", + "total_car_to_hidden_at_8": 1.9575153104960918 }, { + "base_success": 0.0, + "base_utility": 0.050487834960222244, "branch_car_at_8": 0.13774145022034645, "chart_id": "ms-0345d8ac530f88cefd61", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10996723271348506, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.18969689309597015, + "proposal_oracle_utility_gain_over_base_at_8": 0.1392090581357479, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0519554428756237, + "selected_utility_gain_over_base_at_8": 0.0014676079154014587, "selector_regret_at_8": 0.13774145022034645, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1278543956577778 }, { + "base_success": 0.0, + "base_utility": 0.07743936777114868, "branch_car_at_8": 0.004675229080021381, "chart_id": "ms-0819eb89e627595e7e1b", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.14572584629058838, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11537806751098544, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.012503445148468018, + "proposal_oracle_utility_gain_over_base_at_8": -0.06493592262268066, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007828216068446636, + "selected_utility_gain_over_base_at_8": -0.06961115170270205, "selector_regret_at_8": 0.004675229080021381, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13322240114212036, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13789763022214174 }, { + "base_success": 0.0, + "base_utility": 0.19400137662887573, "branch_car_at_8": 0.04213055223226547, "chart_id": "ms-0dee962a5901fb8b5bbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.025648459676823474, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.16528861224651337, + "proposal_oracle_utility_gain_over_base_at_8": -0.028712764382362366, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1231580600142479, + "selected_utility_gain_over_base_at_8": -0.07084331661462784, "selector_regret_at_8": 0.04213055223226547, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8347113877534866, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.876841939985752 }, { + "base_success": 0.0, + "base_utility": 0.04583283141255379, "branch_car_at_8": 0.06697340309619904, "chart_id": "ms-126eb8b36769f9ba4671", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1985912322998047, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.04387270095990371, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.14882762730121613, + "proposal_oracle_utility_gain_over_base_at_8": 0.10299479588866234, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08185422420501709, + "selected_utility_gain_over_base_at_8": 0.0360213927924633, "selector_regret_at_8": 0.06697340309619904, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.04976360499858856, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1167370080947876 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-16661b2fd2c85f57a66c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.5994816934153381, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.00047226750757545233, "chart_id": "ms-20e1efbb745fd02abc95", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14027669602920534, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0017900466918945312, + "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055, "seed": "1375492786", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001317779184319079, + "selected_utility_gain_over_base_at_8": -1.998682220815681, "selector_regret_at_8": 0.00047226750757545233, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2416609227657318, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24213319027330726 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-2dc5138251afd26a040d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565621826", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08252689987421036, "branch_car_at_8": 0.07240533083677292, "chart_id": "ms-387e6771a55be98f1a59", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20744429528713226, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11885014434276354, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.14194543659687042, + "proposal_oracle_utility_gain_over_base_at_8": 0.059418536722660065, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0695401057600975, + "selected_utility_gain_over_base_at_8": -0.012986794114112854, "selector_regret_at_8": 0.07240533083677292, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.06549885869026184, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13790418952703476 }, { + "base_success": 0.0, + "base_utility": 0.03917040303349495, "branch_car_at_8": 0.011704225093126297, "chart_id": "ms-38f7298184a2c5005e94", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1767096221446991, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2012286798976311, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.052575647830963135, + "proposal_oracle_utility_gain_over_base_at_8": 0.013405244797468185, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04087142273783684, + "selected_utility_gain_over_base_at_8": 0.0017010197043418884, "selector_regret_at_8": 0.011704225093126297, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.12413397431373596, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13583819940686226 }, { + "base_success": 0.0, + "base_utility": 0.009124159812927246, "branch_car_at_8": 0.002153436653316021, "chart_id": "ms-482c5c8d208b5907a2c2", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1834145486354828, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1426465578375924, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.008332650177180767, + "proposal_oracle_utility_gain_over_base_at_8": -0.000791509635746479, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006179213523864746, + "selected_utility_gain_over_base_at_8": -0.0029449462890625, "selector_regret_at_8": 0.002153436653316021, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.17508189845830202, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.17723533511161804 }, { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.027157030999660492, "chart_id": "ms-4da03300d92420e80ed0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3089551740932353, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11134227365255356, + "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645, "seed": "1747178030", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08418524265289307, + "selected_utility_gain_over_base_at_8": -0.06212134659290314, "selector_regret_at_8": 0.027157030999660492, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8886577263474464, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.915814757347107 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.00473870849236846, "chart_id": "ms-57be9ae181c077e7821f", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12392755865705843, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.011638104915618896, + "proposal_oracle_utility_gain_over_base_at_8": -1.988361895084381, "seed": "1549324805", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.006899396423250437, + "selected_utility_gain_over_base_at_8": -1.9931006035767496, "selector_regret_at_8": 0.00473870849236846, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.988361895084381, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9931006035767496 }, { + "base_success": 0.0, + "base_utility": 0.1544450968503952, "branch_car_at_8": 0.006060939282178879, "chart_id": "ms-6388b5d2356c556c3e55", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.018031511869928168, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04087907075881958, + "proposal_oracle_utility_gain_over_base_at_8": -0.11356602609157562, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0348181314766407, + "selected_utility_gain_over_base_at_8": -0.1196269653737545, "selector_regret_at_8": 0.006060939282178879, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9591209292411804, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9651818685233593 }, { + "base_success": 0.0, + "base_utility": 0.06173701956868172, "branch_car_at_8": 0.043069109320640564, "chart_id": "ms-638d2ba1917eff4d09e0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.11176212877035141, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09922946147365272, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1519772857427597, + "proposal_oracle_utility_gain_over_base_at_8": 0.09024026617407799, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10890817642211914, + "selected_utility_gain_over_base_at_8": 0.047171156853437424, "selector_regret_at_8": 0.043069109320640564, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0028539523482322693 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-652ad552ba98d2736620", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2367449700832367, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.03930353879708036, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.010964850895106792, + "proposal_oracle_utility_gain_over_base_at_8": -1.9890351491048932, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.010964850895106792, + "selected_utility_gain_over_base_at_8": -1.9890351491048932, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2257801191881299, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2257801191881299 }, { + "base_success": 0.0, + "base_utility": 0.03784912824630737, "branch_car_at_8": 0.014707427471876144, "chart_id": "ms-674fad0e7c042d23629e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.15573063492774963, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.06740395181889638, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05826536938548088, + "proposal_oracle_utility_gain_over_base_at_8": 0.020416241139173508, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043557941913604736, + "selected_utility_gain_over_base_at_8": 0.005708813667297363, "selector_regret_at_8": 0.014707427471876144, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09746526554226875, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1121726930141449 }, { + "base_success": 0.0, + "base_utility": 0.12791956961154938, "branch_car_at_8": 0.041699863970279694, "chart_id": "ms-728b865677134d3788e3", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.22658847272396088, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.36236752551402057, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06523416191339493, + "proposal_oracle_utility_gain_over_base_at_8": -0.06268540769815445, "seed": "750448296", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.023534297943115234, + "selected_utility_gain_over_base_at_8": -0.10438527166843414, "selector_regret_at_8": 0.041699863970279694, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16135431081056595, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.20305417478084564 }, { + "base_success": 0.0, + "base_utility": 0.20529556274414062, "branch_car_at_8": 0.03670026361942291, "chart_id": "ms-7ea98b8f4d2220347ee0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.27989473938941956, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.34662446466496566, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1852296143770218, + "proposal_oracle_utility_gain_over_base_at_8": -0.020065948367118835, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14852935075759888, + "selected_utility_gain_over_base_at_8": -0.05676621198654175, "selector_regret_at_8": 0.03670026361942291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09466512501239777, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13136538863182068 }, { + "base_success": 0.0, + "base_utility": 0.18213583528995514, "branch_car_at_8": 0.0040488241938874125, "chart_id": "ms-7f57045c114ddad153b5", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.227622389793396, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08721578826990975, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.005460778716951609, + "proposal_oracle_utility_gain_over_base_at_8": -0.17667505657300353, "seed": "278008524", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0014119545230641961, + "selected_utility_gain_over_base_at_8": -0.18072388076689094, "selector_regret_at_8": 0.0040488241938874125, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2221616110764444, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2262104352703318 }, { + "base_success": 0.0, + "base_utility": 0.0056487321853637695, "branch_car_at_8": 0.010425567161291838, "chart_id": "ms-81f4e8858b2ba90aea2e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1631626933813095, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.19758543362123893, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.014848490245640278, + "proposal_oracle_utility_gain_over_base_at_8": 0.009199758060276508, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.00442292308434844, + "selected_utility_gain_over_base_at_8": -0.0012258091010153294, "selector_regret_at_8": 0.010425567161291838, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.14831420313566923, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15873977029696107 }, { + "base_success": 0.0, + "base_utility": 0.00698717450723052, "branch_car_at_8": 0.07316120713949203, "chart_id": "ms-8a72b73ac452bc190aeb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.10329639911651611, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.44087366875059886, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11383946985006332, + "proposal_oracle_utility_gain_over_base_at_8": 0.1068522953428328, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04067826271057129, + "selected_utility_gain_over_base_at_8": 0.03369108820334077, "selector_regret_at_8": 0.07316120713949203, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.06261813640594482 }, { + "base_success": 0.0, + "base_utility": 0.08285123109817505, "branch_car_at_8": 0.034163929522037506, "chart_id": "ms-a96bca1ef0b74f686018", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1572066992521286, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0563221293249205, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08467630296945572, + "proposal_oracle_utility_gain_over_base_at_8": 0.0018250718712806702, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05051237344741821, + "selected_utility_gain_over_base_at_8": -0.032338857650756836, "selector_regret_at_8": 0.034163929522037506, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.07253039628267288, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.10669432580471039 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.05995168536901474, "chart_id": "ms-ac4fb33f3ab4db853995", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0474766148185784, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.17047619819641113, + "proposal_oracle_utility_gain_over_base_at_8": -1.8295238018035889, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.11052451282739639, + "selected_utility_gain_over_base_at_8": -1.8894754871726036, "selector_regret_at_8": 0.05995168536901474, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8295238018035889, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8894754871726036 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c31b98b3d3702c6f9c8c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "2054899126", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.037537217140197754, "branch_car_at_8": 0.0029224762693047523, "chart_id": "ms-c832439976a37f52d094", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2594742774963379, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.011460382589838352, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01820494793355465, + "proposal_oracle_utility_gain_over_base_at_8": -0.019332269206643105, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.015282471664249897, + "selected_utility_gain_over_base_at_8": -0.022254745475947857, "selector_regret_at_8": 0.0029224762693047523, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24126932956278324, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.244191805832088 }, { + "base_success": 0.0, + "base_utility": 0.09138330072164536, "branch_car_at_8": 0.008578261360526085, "chart_id": "ms-c96b415f083f60857334", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.26426148414611816, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.06282513857710854, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.024076303467154503, + "proposal_oracle_utility_gain_over_base_at_8": -0.06730699725449085, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.015498042106628418, + "selected_utility_gain_over_base_at_8": -0.07588525861501694, "selector_regret_at_8": 0.008578261360526085, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24018518067896366, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24876344203948975 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9558932781219482, "chart_id": "ms-d73fd5fce55fb58ef8e9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14369576234023818, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "534717141", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.04410672187805176, + "selected_utility_gain_over_base_at_8": -1.9558932781219482, "selector_regret_at_8": 1.9558932781219482, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9558932781219482 }, { + "base_success": 0.0, + "base_utility": 0.14809735119342804, "branch_car_at_8": 0.0, "chart_id": "ms-da142025733b53fcf7eb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2643269896507263, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08970759701490164, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0028313398361206055, + "proposal_oracle_utility_gain_over_base_at_8": -0.14526601135730743, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0028313398361206055, + "selected_utility_gain_over_base_at_8": -0.14526601135730743, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2614956498146057, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2614956498146057 }, { + "base_success": 0.0, + "base_utility": 0.1362423449754715, "branch_car_at_8": 0.0, "chart_id": "ms-e2bad6e99e8a281207cc", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20825156569480896, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13534544179994237, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.023882150650024414, + "proposal_oracle_utility_gain_over_base_at_8": -0.11236019432544708, "seed": "1549324805", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.023882150650024414, + "selected_utility_gain_over_base_at_8": -0.11236019432544708, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.18436941504478455, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.18436941504478455 }, { + "base_success": 0.0, + "base_utility": 0.09695577621459961, "branch_car_at_8": 0.024704236537218094, "chart_id": "ms-e30fd80adc089e9f697a", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18920521438121796, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14400407985489305, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0564744658768177, + "proposal_oracle_utility_gain_over_base_at_8": -0.040481310337781906, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03177022933959961, + "selected_utility_gain_over_base_at_8": -0.065185546875, "selector_regret_at_8": 0.024704236537218094, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13273074850440025, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15743498504161835 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.014290884137153625, "chart_id": "ms-e9ed5d488afdfd6449d1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18713029784839189, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.15760482847690582, + "proposal_oracle_utility_gain_over_base_at_8": -1.8423951715230942, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.1433139443397522, + "selected_utility_gain_over_base_at_8": -1.8566860556602478, "selector_regret_at_8": 0.014290884137153625, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8423951715230942, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8566860556602478 }, { + "base_success": 0.0, + "base_utility": 0.09514755010604858, "branch_car_at_8": 0.012320853769779205, "chart_id": "ms-ec013396e4aff6170ee1", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24502670764923096, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.10651513414126283, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08161912113428116, + "proposal_oracle_utility_gain_over_base_at_8": -0.013528428971767426, "seed": "750448296", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06929826736450195, + "selected_utility_gain_over_base_at_8": -0.02584928274154663, "selector_regret_at_8": 0.012320853769779205, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1634075865149498, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.175728440284729 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-ee30ebaf98622759d888", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07628525991805817, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.013928472995758057, + "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242, "seed": "636680137", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013928472995758057, + "selected_utility_gain_over_base_at_8": -1.986071527004242, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.986071527004242, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.986071527004242 }, { + "base_success": 0.0, + "base_utility": 0.026694953441619873, "branch_car_at_8": 0.073831457644701, "chart_id": "ms-f3f1d07cfd59b823da47", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.17124073207378387, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.41381875369242904, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11723694950342178, + "proposal_oracle_utility_gain_over_base_at_8": 0.09054199606180191, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04340549185872078, + "selected_utility_gain_over_base_at_8": 0.016710538417100906, "selector_regret_at_8": 0.073831457644701, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.05400378257036209, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1278352402150631 }, { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.019766509532928467, "chart_id": "ms-081952a63087fc8f83cd", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07786951284417709, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09589323401451111, + "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298, "seed": "2141802816", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07612672448158264, + "selected_utility_gain_over_base_at_8": 0.005319476127624512, "selector_regret_at_8": 0.019766509532928467, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.904106765985489, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9238732755184174 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-121bc2e6676c1ac10911", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.389114397227515, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "77147264", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-22f153f0d4fa656018e6", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1271718007", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.009028241038322449, "chart_id": "ms-27c9f70e97b8850cc3ab", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13674585276395504, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.034886181354522705, + "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067, "seed": "1881223807", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.025857940316200256, + "selected_utility_gain_over_base_at_8": -0.006169050931930542, "selector_regret_at_8": 0.009028241038322449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9651138186454773, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9741420596837997 }, { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, "chart_id": "ms-2867deaca7e98582620b", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2180235973132697, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, "seed": "1189392917", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, "chart_id": "ms-365d8c6de6de16f66744", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2554442550173647, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, "seed": "1997090716", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03280790150165558, "branch_car_at_8": 0.01877032220363617, "chart_id": "ms-3c326d05fec66e1d6bea", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.40489593467151774, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05748283863067627, + "proposal_oracle_utility_gain_over_base_at_8": 0.02467493712902069, "seed": "278008524", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0387125164270401, + "selected_utility_gain_over_base_at_8": 0.0059046149253845215, "selector_regret_at_8": 0.01877032220363617, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9425171613693237, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.96128748357296 }, { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, "chart_id": "ms-4dbdc6ed61dccebbfe92", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28339362113630795, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, "seed": "794181626", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-58950fed04d203b97ab9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.04977883974023356, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1113371261", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9307887554168701, "chart_id": "ms-651bc0395d78d2819158", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13752192684190806, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1483645336", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.06921124458312988, + "selected_utility_gain_over_base_at_8": -1.9307887554168701, "selector_regret_at_8": 1.9307887554168701, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9307887554168701 }, { + "base_success": 0.0, + "base_utility": 0.058033183217048645, "branch_car_at_8": 1.974869817495346, "chart_id": "ms-9fad82fd288e4296e354", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1815411965846294, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514, "seed": "1176943004", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02513018250465393, + "selected_utility_gain_over_base_at_8": -0.032903000712394714, "selector_regret_at_8": 1.974869817495346, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.974869817495346 }, { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.9512543678283691, "chart_id": "ms-b8bfdd3e6c31bd98082d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28991947553571784, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, "seed": "998761162", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04874563217163086, + "selected_utility_gain_over_base_at_8": -0.019070938229560852, "selector_regret_at_8": 1.9512543678283691, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9512543678283691 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9863876849412918, "chart_id": "ms-fc8084d5cd7da937fc02", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1306768270010748, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1991414747", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013612315058708191, + "selected_utility_gain_over_base_at_8": -1.9863876849412918, "selector_regret_at_8": 1.9863876849412918, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9863876849412918 }, { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.5863077491521835, "chart_id": "ms-3afd8959363e917591d4", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18559462278331035, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8128874897956848, + "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579, "seed": "75070641", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22657974064350128, + "selected_utility_gain_over_base_at_8": 0.02382810413837433, "selector_regret_at_8": 0.5863077491521835, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1871125102043152, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7734202593564987 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.8253512382507324, "chart_id": "ms-3ff1aedb72f65a182b58", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.318234334809661, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1055567618", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.17464876174926758, + "selected_utility_gain_over_base_at_8": -1.8253512382507324, "selector_regret_at_8": 1.8253512382507324, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8253512382507324 }, { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 0.6370042562484741, "chart_id": "ms-4048f8f6e047422e3c79", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07450750352404123, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8232162594795227, + "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314, "seed": "1055157354", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18621200323104858, + "selected_utility_gain_over_base_at_8": 0.009720906615257263, "selector_regret_at_8": 0.6370042562484741, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1767837405204773, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8137879967689514 }, { + "base_success": 0.0, + "base_utility": 0.19844311475753784, "branch_car_at_8": 0.03616611659526825, "chart_id": "ms-5b26c33feed6992386b1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28555947838938045, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.23248937726020813, + "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029, "seed": "1522272686", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.19632326066493988, + "selected_utility_gain_over_base_at_8": -0.0021198540925979614, "selector_regret_at_8": 0.03616611659526825, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7675106227397919, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8036767393350601 }, { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.6792904287576675, "chart_id": "ms-63ac245579b1188935ad", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07548939908426067, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8358442187309265, + "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215, "seed": "211245842", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15655378997325897, + "selected_utility_gain_over_base_at_8": -0.6822806149721146, "selector_regret_at_8": 0.6792904287576675, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1641557812690735, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.843446210026741 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-6c8d06bace98a28a8e4e", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1665739511", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.7044661641120911, "chart_id": "ms-72777024349c2aacd5e0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.03135306008748828, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8125, + "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748, "seed": "256770874", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10803383588790894, + "selected_utility_gain_over_base_at_8": -0.7477142214775085, "selector_regret_at_8": 0.7044661641120911, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1875, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.891966164112091 }, { + "base_success": 0.0, + "base_utility": 0.20372186601161957, "branch_car_at_8": 1.7757539749145508, "chart_id": "ms-7fc5696cb3dc7b9bb9f7", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07643021229779011, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804, "seed": "957101489", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22424602508544922, + "selected_utility_gain_over_base_at_8": 0.02052415907382965, "selector_regret_at_8": 1.7757539749145508, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7757539749145508 }, { + "base_success": 0.0, + "base_utility": 0.8431589603424072, "branch_car_at_8": 0.0, "chart_id": "ms-9a14552b7db695dee92c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13704637411112214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8532722592353821, + "proposal_oracle_utility_gain_over_base_at_8": 0.010113298892974854, "seed": "77147264", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8532722592353821, + "selected_utility_gain_over_base_at_8": 0.010113298892974854, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.146727740764618, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.146727740764618 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.6036040186882019, "chart_id": "ms-9f5d57fe2153fd267774", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18975917078230178, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8267073631286621, + "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338, "seed": "1607773860", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.2231033444404602, + "selected_utility_gain_over_base_at_8": -1.7768966555595398, "selector_regret_at_8": 0.6036040186882019, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.173292636871338, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7768966555595398 }, { + "base_success": 0.0, + "base_utility": 0.8324764966964722, "branch_car_at_8": 0.6467066705226898, "chart_id": "ms-b0fee26e1db967f90b81", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046694794435454434, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8275434374809265, + "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654, "seed": "1358421325", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1808367669582367, + "selected_utility_gain_over_base_at_8": -0.6516397297382355, "selector_regret_at_8": 0.6467066705226898, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1724565625190735, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8191632330417633 }, { + "base_success": 0.0, + "base_utility": 0.8307622671127319, "branch_car_at_8": 0.0070628821849823, "chart_id": "ms-bc2d3b425d9163a62dbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14047863046133235, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22209535539150238, + "proposal_oracle_utility_gain_over_base_at_8": -0.6086669117212296, "seed": "1271718007", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21503247320652008, + "selected_utility_gain_over_base_at_8": -0.6157297939062119, "selector_regret_at_8": 0.0070628821849823, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7779046446084976, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.78496752679348 }, { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.7668711841106415, "chart_id": "ms-d1960368ef76bbb10669", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07978206783428046, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, "seed": "416183106", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23312881588935852, + "selected_utility_gain_over_base_at_8": 0.03280240297317505, "selector_regret_at_8": 1.7668711841106415, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7668711841106415 }, { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, "chart_id": "ms-efdc07c985935d871adf", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.16443847051324711, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, "seed": "706479600", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.0, "chart_id": "ms-fd2387aeed9bfc9a0d08", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13114443223224567, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.046244338154792786, + "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537, "seed": "1919198188", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046244338154792786, + "selected_utility_gain_over_base_at_8": 0.009045466780662537, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9537556618452072, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9537556618452072 } ], "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4521276595744681, + "low": 0.20921985815602834, + "mean": 0.3297872340425532, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3297872340425532, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.33815987933634994, + "low": 0.10666666666666666, + "mean": 0.24241327300150828, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24241327300150828, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.42028985507246375, + "low": 0.18840579710144928, + "mean": 0.2898550724637681, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0927389474422176, + "low": 0.6324926499567979, + "mean": 0.8566858090574239, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8566858090574239, + "per_seed": { + "1025740212": 0.528373658657074, + "1055157354": 0.17649109661579132, + "1055567618": 2.0, + "1113268940": 0.42231252789497375, + "1113371261": 2.0, + "1176943004": 0.058033183217048645, + "1189392917": 0.10841864347457886, + "1231780997": 0.7270061373710632, + "1271718007": 1.415381133556366, + "1273247072": 0.15342965722084045, + "1358421325": 0.8324764966964722, + "1375492786": 2.0, + "141984615": 0.09381213908394177, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.19844311475753784, + "1528740417": 2.0, + "1549324805": 1.3787474483251572, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.1463065892457962, + "1881223807": 0.0320269912481308, + "1919198188": 0.03719887137413025, + "1991414747": 2.0, + "1997090716": 0.13927479088306427, + "2015956957": 0.06711978986859321, + "2054899126": 2.0, + "211245842": 0.8388344049453735, + "2141802816": 0.07080724835395813, + "256770874": 0.8557480573654175, + "278008524": 0.10747186839580536, + "416183106": 0.20032641291618347, + "534717141": 2.0, + "551504635": 0.7247228622436523, + "565369392": 0.7997656464576721, + "565621826": 2.0, + "585904130": 0.07529209802548091, + "626486813": 0.5054400166263804, + "636680137": 2.0, + "706479600": 0.8350887894630432, + "750448296": 0.11153355985879898, + "75070641": 0.20275163650512695, + "767747424": 0.5475241877138615, + "77147264": 1.4215794801712036, + "794181626": 0.18095605075359344, + "957101489": 0.20372186601161957, + "998761162": 0.06781657040119171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9312208624280917, + "low": 0.4135245136419933, + "mean": 0.6995428283303713, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6995428283303713, + "per_task": { + "LiftPegUpright-v1": 1.0670691976944606, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.8223185676794785, + "StackCube-v1": 0.8036667982737223 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9620560415399572, + "low": 0.5693137269615587, + "mean": 0.7455479058545029, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, "branch_car_at_8": { "macro_by_seed": { "ci": { @@ -855,6 +2139,186 @@ }, "n": 69 }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.922695035460993, + "low": 0.7351063829787235, + "mean": 0.8358156028368794, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8358156028368794, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.782608695652174, + "low": 0.5507246376811594, + "mean": 0.6666666666666666, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8615907955016442, + "low": 1.529367521088174, + "mean": 1.7062028111638965, + "n": 47, + "num_samples": 500 + }, + "mean": 1.7062028111638965, + "per_seed": { + "1025740212": 2.0, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 2.0, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1358421325": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.20044784247875214, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 2.0, + "1528740417": 2.0, + "1549324805": 1.4027505218982697, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 2.0, + "1881223807": 2.0, + "1919198188": 2.0, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.5298813670873642, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 2.0, + "278008524": 1.113811194896698, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 1.4266315797964733, + "565621826": 2.0, + "585904130": 0.23764699200789133, + "626486813": 0.6124684102833271, + "636680137": 2.0, + "706479600": 2.0, + "750448296": 0.23580759018659592, + "75070641": 2.0, + "767747424": 0.18863565661013126, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.6062410234109215, + "low": 1.1855228276572365, + "mean": 1.3999846732054932, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, "outcome_ptr_at_8": { "macro_by_seed": { "ci": { @@ -1035,134 +2499,1124 @@ }, "n": 69 }, - "selector_regret_at_8": { + "proposal_oracle_success_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.6905135366465427, - "low": 0.2873830998237971, - "mean": 0.46333538608845154, + "high": 0.5673758865248226, + "low": 0.2943262411347518, + "mean": 0.4326241134751774, "n": 47, "num_samples": 500 }, - "mean": 0.46333538608845154, + "mean": 0.4326241134751774, "per_seed": { - "1025740212": 0.47739143669605255, - "1055157354": 0.6370042562484741, - "1055567618": 1.8253512382507324, - "1113268940": 0.18306860327720642, - "1113371261": 0.0, - "1176943004": 1.974869817495346, - "1189392917": 0.0, - "1231780997": 0.0, - "1271718007": 0.00353144109249115, - "1273247072": 0.45726464316248894, - "1358421325": 0.6467066705226898, - "1375492786": 0.00047226750757545233, - "141984615": 0.05730179324746132, - "1475440973": 1.289921224117279, - "1483645336": 1.9307887554168701, - "1522272686": 0.03616611659526825, - "1528740417": 0.0, - "1549324805": 0.0015795694974561532, - "1607773860": 0.6036040186882019, - "1665739511": 0.0, - "1747178030": 0.027157030999660492, - "1881223807": 0.009028241038322449, + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, "1919198188": 0.0, - "1991414747": 1.9863876849412918, - "1997090716": 0.0, - "2015956957": 0.022195855714380743, - "2054899126": 0.0, - "211245842": 0.6792904287576675, - "2141802816": 0.019766509532928467, - "256770874": 0.7044661641120911, - "278008524": 0.011409573198761791, - "416183106": 1.7668711841106415, - "534717141": 1.9558932781219482, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 1.0, "551504635": 0.0, - "565369392": 0.046260833740234375, - "565621826": 0.0, - "585904130": 0.012068324722349644, - "626486813": 0.02500777377281338, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, "636680137": 0.0, - "706479600": 0.0, - "750448296": 0.02701035887002945, - "75070641": 0.5863077491521835, - "767747424": 0.04561196081340313, - "77147264": 0.0, - "794181626": 0.0, - "957101489": 1.7757539749145508, - "998761162": 1.9512543678283691 + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 } }, "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.5835942497964088, - "low": 0.22782261933967035, - "mean": 0.4178369468370689, + "high": 0.5850678733031673, + "low": 0.1, + "mean": 0.3440422322775264, "n": 5, "num_samples": 500 }, - "mean": 0.4178369468370689, + "mean": 0.3440422322775264, "per_task": { - "LiftPegUpright-v1": 0.3250635440150897, - "PickCube-v1": 0.45726464316248894, - "PullCube-v1": 0.08196123232654132, - "PushCube-v1": 0.6069896691120588, - "StackCube-v1": 0.6179056455691655 + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 0.0, + "PullCube-v1": 0.11764705882352941, + "PushCube-v1": 0.7692307692307693, + "StackCube-v1": 0.3333333333333333 } }, - "metric": "selector_regret_at_8", + "metric": "proposal_oracle_success_at_8", "micro": { "confidence": 0.95, - "high": 0.48046939117341314, - "low": 0.18767418079685583, - "mean": 0.32396779983115476, + "high": 0.43478260869565216, + "low": 0.21739130434782608, + "mean": 0.3188405797101449, "n": 69, "num_samples": 500 }, "n": 69 }, - "support_gap_at_8": { + "proposal_oracle_success_gain_over_base_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.8704533781212297, - "low": 0.4250871043820726, - "mean": 0.6443930889275058, + "high": 0.25709219858156024, + "low": -0.03368794326241135, + "mean": 0.10283687943262412, "n": 47, "num_samples": 500 }, - "mean": 0.6443930889275058, + "mean": 0.10283687943262412, "per_seed": { - "1025740212": 1.3016598224639893, - "1055157354": 1.1767837405204773, + "1025740212": 0.0, + "1055157354": 0.0, "1055567618": 0.0, - "1113268940": 1.33825021982193, + "1113268940": 0.0, "1113371261": 0.0, - "1176943004": 0.0, - "1189392917": 0.0, - "1231780997": 0.0, - "1271718007": 0.8889523223042488, - "1273247072": 1.500250667333603, - "1358421325": 1.1724565625190735, - "1375492786": 0.2416609227657318, - "141984615": 0.11134201536575954, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, "1475440973": 0.0, "1483645336": 0.0, - "1522272686": 1.7675106227397919, + "1522272686": 0.0, "1528740417": 0.0, - "1549324805": 0.7242437700430552, - "1607773860": 1.173292636871338, + "1549324805": -0.3333333333333333, + "1607773860": -1.0, "1665739511": 0.0, - "1747178030": 1.8886577263474464, - "1881223807": 1.9651138186454773, - "1919198188": 1.9537556618452072, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, "1991414747": 0.0, - "1997090716": 0.0, - "2015956957": 0.4735892705619335, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2641025641025641, + "low": -0.07058823529411765, + "mean": 0.10162895927601809, + "n": 5, + "num_samples": 500 + }, + "mean": 0.10162895927601809, + "per_task": { + "LiftPegUpright-v1": 0.16666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": -0.17647058823529413, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.13333333333333333 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.14492753623188406, + "low": -0.08695652173913043, + "mean": 0.028985507246376812, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.318000317976896, + "low": 0.8185871909540939, + "mean": 1.0621498337636393, + "n": 47, + "num_samples": 500 + }, + "mean": 1.0621498337636393, + "per_seed": { + "1025740212": 0.6983401775360107, + "1055157354": 0.8232162594795227, + "1055567618": 2.0, + "1113268940": 0.6617497801780701, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 1.1110476776957512, + "1273247072": 0.4997493326663971, + "1358421325": 0.8275434374809265, + "1375492786": 0.0017900466918945312, + "141984615": 0.09240151196718216, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.23248937726020813, + "1528740417": 2.0, + "1549324805": 0.6785067518552145, + "1607773860": 0.8267073631286621, + "1665739511": 2.0, + "1747178030": 0.11134227365255356, + "1881223807": 0.034886181354522705, + "1919198188": 0.046244338154792786, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.05629209652543068, + "2054899126": 2.0, + "211245842": 0.8358442187309265, + "2141802816": 0.09589323401451111, + "256770874": 0.8125, + "278008524": 0.03147180867381394, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 0.7224790453910828, + "565369392": 0.17366480827331543, + "565621826": 2.0, + "585904130": 0.032918572425842285, + "626486813": 0.07365635968744755, + "636680137": 0.013928472995758057, + "706479600": 2.0, + "750448296": 0.07342664152383804, + "75070641": 0.8128874897956848, + "767747424": 0.11342880013398826, + "77147264": 1.426636129617691, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3423044667794155, + "low": 0.537627078428061, + "mean": 0.9574398233701207, + "n": 5, + "num_samples": 500 + }, + "mean": 0.9574398233701207, + "per_task": { + "LiftPegUpright-v1": 1.3470948338508606, + "PickCube-v1": 0.4997493326663971, + "PullCube-v1": 0.3012250267818351, + "PushCube-v1": 1.55294325030767, + "StackCube-v1": 1.0861866732438406 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.0082228477517872, + "low": 0.6154177648528223, + "mean": 0.8015221970871199, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4860803771314892, + "low": -0.031156662437926107, + "mean": 0.20546402470621525, + "n": 47, + "num_samples": 500 + }, + "mean": 0.20546402470621525, + "per_seed": { + "1025740212": 0.16996651887893677, + "1055157354": 0.6467251628637314, + "1055567618": 0.0, + "1113268940": 0.2394372522830963, + "1113371261": 0.0, + "1176943004": 1.9419668167829514, + "1189392917": 1.8915813565254211, + "1231780997": 1.2729938626289368, + "1271718007": -0.3043334558606148, + "1273247072": 0.34631967544555664, + "1358421325": -0.004933059215545654, + "1375492786": -1.9982099533081055, + "141984615": -0.001410627116759618, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.03404626250267029, + "1528740417": 0.0, + "1549324805": -0.7002406964699427, + "1607773860": -1.173292636871338, + "1665739511": 0.0, + "1747178030": -0.034964315593242645, + "1881223807": 0.0028591901063919067, + "1919198188": 0.009045466780662537, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": -0.010827693343162536, + "2054899126": 0.0, + "211245842": -0.0029901862144470215, + "2141802816": 0.02508598566055298, + "256770874": -0.04324805736541748, + "278008524": -0.07600005972199142, + "416183106": 1.7996735870838165, + "534717141": 0.0, + "551504635": -0.00224381685256958, + "565369392": -0.6261008381843567, + "565621826": 0.0, + "585904130": -0.04237352559963862, + "626486813": -0.43178365693893284, + "636680137": -1.986071527004242, + "706479600": 1.1649112105369568, + "750448296": -0.03810691833496094, + "75070641": 0.6101358532905579, + "767747424": -0.4340953875798732, + "77147264": 0.005056649446487427, + "794181626": 1.8190439492464066, + "957101489": 1.7962781339883804, + "998761162": 1.9321834295988083 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5508839118022185, + "low": -0.08473387408052005, + "mean": 0.2578969950397495, + "n": 5, + "num_samples": 500 + }, + "mean": 0.2578969950397495, + "per_task": { + "LiftPegUpright-v1": 0.2800256361564, + "PickCube-v1": 0.34631967544555664, + "PullCube-v1": -0.35000489400151896, + "PushCube-v1": 0.7306246826281915, + "StackCube-v1": 0.2825198749701182 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2701519333851942, + "low": -0.15554412837018786, + "mean": 0.055974291232617004, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3475177304964539, + "low": 0.1347517730496454, + "mean": 0.24113475177304963, + "n": 47, + "num_samples": 500 + }, + "mean": 0.24113475177304963, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3589743589743589, + "low": 0.05333333333333333, + "mean": 0.20328808446455504, + "n": 5, + "num_samples": 500 + }, + "mean": 0.20328808446455504, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08823529411764706, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.13333333333333333 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2898550724637681, + "low": 0.11594202898550725, + "mean": 0.18840579710144928, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.04609929078014185, + "low": -0.23404255319148937, + "mean": -0.08865248226950355, + "n": 47, + "num_samples": 500 + }, + "mean": -0.08865248226950355, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": -1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": -1.0, + "1483645336": -1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": -0.3333333333333333, + "1607773860": -1.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": -1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": -1.0, + "551504635": 0.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.03282051282051283, + "low": -0.13686274509803922, + "mean": -0.03912518853695324, + "n": 5, + "num_samples": 500 + }, + "mean": -0.03912518853695324, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.20588235294117646, + "PushCube-v1": 0.07692307692307693, + "StackCube-v1": -0.06666666666666667 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.0, + "low": -0.21739130434782608, + "mean": -0.10144927536231885, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7957249261758369, + "low": 0.38785451346428745, + "mean": 0.5988144476751877, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5988144476751877, + "per_seed": { + "1025740212": 0.2209487408399582, + "1055157354": 0.18621200323104858, + "1055567618": 0.17464876174926758, + "1113268940": 0.47868117690086365, + "1113371261": 2.0, + "1176943004": 0.02513018250465393, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 1.10751623660326, + "1273247072": 0.04248468950390816, + "1358421325": 0.1808367669582367, + "1375492786": 0.001317779184319079, + "141984615": 0.03509971871972084, + "1475440973": 0.710078775882721, + "1483645336": 0.06921124458312988, + "1522272686": 0.19632326066493988, + "1528740417": 2.0, + "1549324805": 0.6769271823577583, + "1607773860": 0.2231033444404602, + "1665739511": 2.0, + "1747178030": 0.08418524265289307, + "1881223807": 0.025857940316200256, + "1919198188": 0.046244338154792786, + "1991414747": 0.013612315058708191, + "1997090716": 2.0, + "2015956957": 0.034096240811049935, + "2054899126": 2.0, + "211245842": 0.15655378997325897, + "2141802816": 0.07612672448158264, + "256770874": 0.10803383588790894, + "278008524": 0.020062235475052148, + "416183106": 0.23312881588935852, + "534717141": 0.04410672187805176, + "551504635": 0.7224790453910828, + "565369392": 0.12740397453308105, + "565621826": 2.0, + "585904130": 0.02085024770349264, + "626486813": 0.04864858591463417, + "636680137": 0.013928472995758057, + "706479600": 2.0, + "750448296": 0.046416282653808594, + "75070641": 0.22657974064350128, + "767747424": 0.06781683932058513, + "77147264": 1.426636129617691, + "794181626": 2.0, + "957101489": 0.22424602508544922, + "998761162": 0.04874563217163086 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8462622490316436, + "low": 0.19835559911861575, + "mean": 0.5396028765330518, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5396028765330518, + "per_task": { + "LiftPegUpright-v1": 1.0220312898357709, + "PickCube-v1": 0.04248468950390816, + "PullCube-v1": 0.2192637944552938, + "PushCube-v1": 0.9459535811956112, + "StackCube-v1": 0.468281027674675 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.6843658976587534, + "low": 0.3229842852494018, + "mean": 0.4775543972559651, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.010564623637940616, + "low": -0.5054863751322652, + "mean": -0.25787136138223626, + "n": 47, + "num_samples": 500 + }, + "mean": -0.25787136138223626, + "per_seed": { + "1025740212": -0.3074249178171158, + "1055157354": 0.009720906615257263, + "1055567618": -1.8253512382507324, + "1113268940": 0.05636864900588989, + "1113371261": 0.0, + "1176943004": -0.032903000712394714, + "1189392917": 1.8915813565254211, + "1231780997": 1.2729938626289368, + "1271718007": -0.3078648969531059, + "1273247072": -0.1109449677169323, + "1358421325": -0.6516397297382355, + "1375492786": -1.998682220815681, + "141984615": -0.05871242036422094, + "1475440973": -1.289921224117279, + "1483645336": -1.9307887554168701, + "1522272686": -0.0021198540925979614, + "1528740417": 0.0, + "1549324805": -0.7018202659673989, + "1607773860": -1.7768966555595398, + "1665739511": 0.0, + "1747178030": -0.06212134659290314, + "1881223807": -0.006169050931930542, + "1919198188": 0.009045466780662537, + "1991414747": -1.9863876849412918, + "1997090716": 1.8607252091169357, + "2015956957": -0.033023549057543275, + "2054899126": 0.0, + "211245842": -0.6822806149721146, + "2141802816": 0.005319476127624512, + "256770874": -0.7477142214775085, + "278008524": -0.08740963292075321, + "416183106": 0.03280240297317505, + "534717141": -1.9558932781219482, + "551504635": -0.00224381685256958, + "565369392": -0.6723616719245911, + "565621826": 0.0, + "585904130": -0.05444185032198826, + "626486813": -0.4567914307117462, + "636680137": -1.986071527004242, + "706479600": 1.1649112105369568, + "750448296": -0.06511727720499039, + "75070641": 0.02382810413837433, + "767747424": -0.47970734839327633, + "77147264": 0.005056649446487427, + "794181626": 1.8190439492464066, + "957101489": 0.02052415907382965, + "998761162": -0.019070938229560852 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.0019037275818678035, + "low": -0.33526441148838354, + "mean": -0.1599399517973194, + "n": 5, + "num_samples": 500 + }, + "mean": -0.1599399517973194, + "per_task": { + "LiftPegUpright-v1": -0.04503790785868963, + "PickCube-v1": -0.1109449677169323, + "PullCube-v1": -0.4319661263280603, + "PushCube-v1": 0.12363501351613265, + "StackCube-v1": -0.33538577059904734 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": -0.06954942767143897, + "low": -0.4864171527484702, + "mean": -0.26799350859853777, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6905135366465427, + "low": 0.2873830998237971, + "mean": 0.46333538608845154, + "n": 47, + "num_samples": 500 + }, + "mean": 0.46333538608845154, + "per_seed": { + "1025740212": 0.47739143669605255, + "1055157354": 0.6370042562484741, + "1055567618": 1.8253512382507324, + "1113268940": 0.18306860327720642, + "1113371261": 0.0, + "1176943004": 1.974869817495346, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.00353144109249115, + "1273247072": 0.45726464316248894, + "1358421325": 0.6467066705226898, + "1375492786": 0.00047226750757545233, + "141984615": 0.05730179324746132, + "1475440973": 1.289921224117279, + "1483645336": 1.9307887554168701, + "1522272686": 0.03616611659526825, + "1528740417": 0.0, + "1549324805": 0.0015795694974561532, + "1607773860": 0.6036040186882019, + "1665739511": 0.0, + "1747178030": 0.027157030999660492, + "1881223807": 0.009028241038322449, + "1919198188": 0.0, + "1991414747": 1.9863876849412918, + "1997090716": 0.0, + "2015956957": 0.022195855714380743, + "2054899126": 0.0, + "211245842": 0.6792904287576675, + "2141802816": 0.019766509532928467, + "256770874": 0.7044661641120911, + "278008524": 0.011409573198761791, + "416183106": 1.7668711841106415, + "534717141": 1.9558932781219482, + "551504635": 0.0, + "565369392": 0.046260833740234375, + "565621826": 0.0, + "585904130": 0.012068324722349644, + "626486813": 0.02500777377281338, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.02701035887002945, + "75070641": 0.5863077491521835, + "767747424": 0.04561196081340313, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.7757539749145508, + "998761162": 1.9512543678283691 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5835942497964088, + "low": 0.22782261933967035, + "mean": 0.4178369468370689, + "n": 5, + "num_samples": 500 + }, + "mean": 0.4178369468370689, + "per_task": { + "LiftPegUpright-v1": 0.3250635440150897, + "PickCube-v1": 0.45726464316248894, + "PullCube-v1": 0.08196123232654132, + "PushCube-v1": 0.6069896691120588, + "StackCube-v1": 0.6179056455691655 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.48046939117341314, + "low": 0.18767418079685583, + "mean": 0.32396779983115476, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3191489361702128, + "low": 0.0851063829787234, + "mean": 0.19148936170212766, + "n": 47, + "num_samples": 500 + }, + "mean": 0.19148936170212766, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.24307692307692305, + "low": 0.045098039215686274, + "mean": 0.14075414781297133, + "n": 5, + "num_samples": 500 + }, + "mean": 0.14075414781297133, + "per_task": { + "LiftPegUpright-v1": 0.16666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.029411764705882353, + "PushCube-v1": 0.3076923076923077, + "StackCube-v1": 0.2 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.21739130434782608, + "low": 0.057971014492753624, + "mean": 0.13043478260869565, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.551418439716312, + "low": 0.2726950354609929, + "mean": 0.4031914893617021, + "n": 47, + "num_samples": 500 + }, + "mean": 0.4031914893617021, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7794871794871795, + "low": 0.2746606334841629, + "mean": 0.5206636500754148, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5206636500754148, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 1.0, + "PullCube-v1": 0.20588235294117646, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.463768115942029, + "low": 0.2318840579710145, + "mean": 0.34782608695652173, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7304964539007093, + "low": 0.45886524822695035, + "mean": 0.5946808510638298, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5946808510638298, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8810256410256411, + "low": 0.41719457013574657, + "mean": 0.6614177978883861, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6614177978883861, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.23529411764705882, + "PushCube-v1": 0.5384615384615384, + "StackCube-v1": 0.8666666666666667 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5942028985507246, + "low": 0.36231884057971014, + "mean": 0.4782608695652174, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8704533781212297, + "low": 0.4250871043820726, + "mean": 0.6443930889275058, + "n": 47, + "num_samples": 500 + }, + "mean": 0.6443930889275058, + "per_seed": { + "1025740212": 1.3016598224639893, + "1055157354": 1.1767837405204773, + "1055567618": 0.0, + "1113268940": 1.33825021982193, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.8889523223042488, + "1273247072": 1.500250667333603, + "1358421325": 1.1724565625190735, + "1375492786": 0.2416609227657318, + "141984615": 0.11134201536575954, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.7675106227397919, + "1528740417": 0.0, + "1549324805": 0.7242437700430552, + "1607773860": 1.173292636871338, + "1665739511": 0.0, + "1747178030": 1.8886577263474464, + "1881223807": 1.9651138186454773, + "1919198188": 1.9537556618452072, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.4735892705619335, "2054899126": 0.0, "211245842": 1.1641557812690735, "2141802816": 1.904106765985489, @@ -1214,6 +3668,96 @@ "num_samples": 500 }, "n": 69 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.339665490579098, + "low": 0.8604674879511063, + "mean": 1.1073883634887087, + "n": 47, + "num_samples": 500 + }, + "mean": 1.1073883634887087, + "per_seed": { + "1025740212": 1.7790512591600418, + "1055157354": 1.8137879967689514, + "1055567618": 1.8253512382507324, + "1113268940": 1.5213188230991364, + "1113371261": 0.0, + "1176943004": 1.974869817495346, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.89248376339674, + "1273247072": 1.9575153104960918, + "1358421325": 1.8191632330417633, + "1375492786": 0.24213319027330726, + "141984615": 0.1653481237590313, + "1475440973": 1.289921224117279, + "1483645336": 1.9307887554168701, + "1522272686": 1.8036767393350601, + "1528740417": 0.0, + "1549324805": 0.7258233395405114, + "1607773860": 1.7768966555595398, + "1665739511": 0.0, + "1747178030": 1.915814757347107, + "1881223807": 1.9741420596837997, + "1919198188": 1.9537556618452072, + "1991414747": 1.9863876849412918, + "1997090716": 0.0, + "2015956957": 0.49578512627631427, + "2054899126": 0.0, + "211245842": 1.843446210026741, + "2141802816": 1.9238732755184174, + "256770874": 1.891966164112091, + "278008524": 1.0937489594216458, + "416183106": 1.7668711841106415, + "534717141": 1.9558932781219482, + "551504635": 1.2775209546089172, + "565369392": 1.2992276052633922, + "565621826": 0.0, + "585904130": 0.2167967443043987, + "626486813": 0.5638198243686929, + "636680137": 1.986071527004242, + "706479600": 0.0, + "750448296": 0.18939130753278732, + "75070641": 1.7734202593564987, + "767747424": 0.12081881728954613, + "77147264": 0.573363870382309, + "794181626": 0.0, + "957101489": 1.7757539749145508, + "998761162": 1.9512543678283691 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.676446722795566, + "low": 0.8272199847642051, + "mean": 1.2168614908268247, + "n": 5, + "num_samples": 500 + }, + "mean": 1.2168614908268247, + "per_task": { + "LiftPegUpright-v1": 0.977968710164229, + "PickCube-v1": 1.9575153104960918, + "PullCube-v1": 0.5630580423440894, + "PushCube-v1": 1.0540464188043888, + "StackCube-v1": 1.531718972325325 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 1.1262573117906551, + "low": 0.7123256691755133, + "mean": 0.9224302759495281, + "n": 69, + "num_samples": 500 + }, + "n": 69 } }, "thresholds": [ diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_seed.json index f5692c3575635c674f58c17f4633f90839a10413..c8db434f785c6bcd93354109ac6e9a99edcd55da 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_seed.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_seed.json @@ -1,331 +1,1083 @@ { "1025740212": { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.47739143669605255, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12992986623548633, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6983401775360107, + "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2209487408399582, + "selected_utility_gain_over_base_at_8": -0.3074249178171158, "selector_regret_at_8": 0.47739143669605255, - "support_gap_at_8": 1.3016598224639893 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3016598224639893, + "total_car_to_hidden_at_8": 1.7790512591600418 }, "1055157354": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 0.6370042562484741, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07450750352404123, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8232162594795227, + "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18621200323104858, + "selected_utility_gain_over_base_at_8": 0.009720906615257263, "selector_regret_at_8": 0.6370042562484741, - "support_gap_at_8": 1.1767837405204773 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1767837405204773, + "total_car_to_hidden_at_8": 1.8137879967689514 }, "1055567618": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.8253512382507324, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.318234334809661, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.17464876174926758, + "selected_utility_gain_over_base_at_8": -1.8253512382507324, "selector_regret_at_8": 1.8253512382507324, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8253512382507324 }, "1113268940": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.18306860327720642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10725283874677279, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6617497801780701, + "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.47868117690086365, + "selected_utility_gain_over_base_at_8": 0.05636864900588989, "selector_regret_at_8": 0.18306860327720642, - "support_gap_at_8": 1.33825021982193 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.33825021982193, + "total_car_to_hidden_at_8": 1.5213188230991364 }, "1113371261": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.04977883974023356, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1176943004": { + "base_success": 0.0, + "base_utility": 0.058033183217048645, "branch_car_at_8": 1.974869817495346, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1815411965846294, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02513018250465393, + "selected_utility_gain_over_base_at_8": -0.032903000712394714, "selector_regret_at_8": 1.974869817495346, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.974869817495346 }, "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2180235973132697, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1231780997": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28322114525972475, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.2729938626289368, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1271718007": { + "base_success": 0.5, + "base_utility": 1.415381133556366, "branch_car_at_8": 0.00353144109249115, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07023931523066618, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.1110476776957512, + "proposal_oracle_utility_gain_over_base_at_8": -0.3043334558606148, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.10751623660326, + "selected_utility_gain_over_base_at_8": -0.3078648969531059, "selector_regret_at_8": 0.00353144109249115, - "support_gap_at_8": 0.8889523223042488 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.8889523223042488, + "total_car_to_hidden_at_8": 0.89248376339674 }, "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.45726464316248894, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, "selector_regret_at_8": 0.45726464316248894, - "support_gap_at_8": 1.500250667333603 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.500250667333603, + "total_car_to_hidden_at_8": 1.9575153104960918 }, "1358421325": { + "base_success": 0.0, + "base_utility": 0.8324764966964722, "branch_car_at_8": 0.6467066705226898, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046694794435454434, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8275434374809265, + "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1808367669582367, + "selected_utility_gain_over_base_at_8": -0.6516397297382355, "selector_regret_at_8": 0.6467066705226898, - "support_gap_at_8": 1.1724565625190735 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1724565625190735, + "total_car_to_hidden_at_8": 1.8191632330417633 }, "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.00047226750757545233, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14027669602920534, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0017900466918945312, + "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001317779184319079, + "selected_utility_gain_over_base_at_8": -1.998682220815681, "selector_regret_at_8": 0.00047226750757545233, - "support_gap_at_8": 0.2416609227657318 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.2416609227657318, + "total_car_to_hidden_at_8": 0.24213319027330726 }, "141984615": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, "branch_car_at_8": 0.05730179324746132, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.08533231968443573, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09240151196718216, + "proposal_oracle_utility_gain_over_base_at_8": -0.001410627116759618, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03509971871972084, + "selected_utility_gain_over_base_at_8": -0.05871242036422094, "selector_regret_at_8": 0.05730179324746132, - "support_gap_at_8": 0.11134201536575954 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.11134201536575954, + "total_car_to_hidden_at_8": 0.1653481237590313 }, "1475440973": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.289921224117279, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2443552809335613, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.710078775882721, + "selected_utility_gain_over_base_at_8": -1.289921224117279, "selector_regret_at_8": 1.289921224117279, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.289921224117279 }, "1483645336": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9307887554168701, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13752192684190806, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.06921124458312988, + "selected_utility_gain_over_base_at_8": -1.9307887554168701, "selector_regret_at_8": 1.9307887554168701, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9307887554168701 }, "1522272686": { + "base_success": 0.0, + "base_utility": 0.19844311475753784, "branch_car_at_8": 0.03616611659526825, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28555947838938045, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.23248937726020813, + "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.19632326066493988, + "selected_utility_gain_over_base_at_8": -0.0021198540925979614, "selector_regret_at_8": 0.03616611659526825, - "support_gap_at_8": 1.7675106227397919 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7675106227397919, + "total_car_to_hidden_at_8": 1.8036767393350601 }, "1528740417": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2757671830981365, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1549324805": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, "branch_car_at_8": 0.0015795694974561532, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.28625156462411294, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6785067518552145, + "proposal_oracle_utility_gain_over_base_at_8": -0.7002406964699427, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6769271823577583, + "selected_utility_gain_over_base_at_8": -0.7018202659673989, "selector_regret_at_8": 0.0015795694974561532, - "support_gap_at_8": 0.7242437700430552 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7242437700430552, + "total_car_to_hidden_at_8": 0.7258233395405114 }, "1607773860": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.6036040186882019, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18975917078230178, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8267073631286621, + "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.2231033444404602, + "selected_utility_gain_over_base_at_8": -1.7768966555595398, "selector_regret_at_8": 0.6036040186882019, - "support_gap_at_8": 1.173292636871338 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.173292636871338, + "total_car_to_hidden_at_8": 1.7768966555595398 }, "1665739511": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1747178030": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.027157030999660492, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3089551740932353, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11134227365255356, + "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08418524265289307, + "selected_utility_gain_over_base_at_8": -0.06212134659290314, "selector_regret_at_8": 0.027157030999660492, - "support_gap_at_8": 1.8886577263474464 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8886577263474464, + "total_car_to_hidden_at_8": 1.915814757347107 }, "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.009028241038322449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13674585276395504, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.034886181354522705, + "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.025857940316200256, + "selected_utility_gain_over_base_at_8": -0.006169050931930542, "selector_regret_at_8": 0.009028241038322449, - "support_gap_at_8": 1.9651138186454773 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9651138186454773, + "total_car_to_hidden_at_8": 1.9741420596837997 }, "1919198188": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13114443223224567, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.046244338154792786, + "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046244338154792786, + "selected_utility_gain_over_base_at_8": 0.009045466780662537, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.9537556618452072 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9537556618452072, + "total_car_to_hidden_at_8": 1.9537556618452072 }, "1991414747": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9863876849412918, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1306768270010748, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013612315058708191, + "selected_utility_gain_over_base_at_8": -1.9863876849412918, "selector_regret_at_8": 1.9863876849412918, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9863876849412918 }, "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2554442550173647, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2015956957": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, "branch_car_at_8": 0.022195855714380743, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.16317219295797403, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05629209652543068, + "proposal_oracle_utility_gain_over_base_at_8": -0.010827693343162536, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.034096240811049935, + "selected_utility_gain_over_base_at_8": -0.033023549057543275, "selector_regret_at_8": 0.022195855714380743, - "support_gap_at_8": 0.4735892705619335 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.2, + "success_total_car_to_hidden_at_8": 0.2, + "support_gap_at_8": 0.4735892705619335, + "total_car_to_hidden_at_8": 0.49578512627631427 }, "2054899126": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "211245842": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.6792904287576675, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07548939908426067, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8358442187309265, + "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15655378997325897, + "selected_utility_gain_over_base_at_8": -0.6822806149721146, "selector_regret_at_8": 0.6792904287576675, - "support_gap_at_8": 1.1641557812690735 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1641557812690735, + "total_car_to_hidden_at_8": 1.843446210026741 }, "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.019766509532928467, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07786951284417709, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09589323401451111, + "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07612672448158264, + "selected_utility_gain_over_base_at_8": 0.005319476127624512, "selector_regret_at_8": 0.019766509532928467, - "support_gap_at_8": 1.904106765985489 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.904106765985489, + "total_car_to_hidden_at_8": 1.9238732755184174 }, "256770874": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.7044661641120911, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.03135306008748828, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8125, + "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10803383588790894, + "selected_utility_gain_over_base_at_8": -0.7477142214775085, "selector_regret_at_8": 0.7044661641120911, - "support_gap_at_8": 1.1875 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1875, + "total_car_to_hidden_at_8": 1.891966164112091 }, "278008524": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, "branch_car_at_8": 0.011409573198761791, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.24605586147071373, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03147180867381394, + "proposal_oracle_utility_gain_over_base_at_8": -0.07600005972199142, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.020062235475052148, + "selected_utility_gain_over_base_at_8": -0.08740963292075321, "selector_regret_at_8": 0.011409573198761791, - "support_gap_at_8": 1.082339386222884 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.082339386222884, + "total_car_to_hidden_at_8": 1.0937489594216458 }, "416183106": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.7668711841106415, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07978206783428046, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23312881588935852, + "selected_utility_gain_over_base_at_8": 0.03280240297317505, "selector_regret_at_8": 1.7668711841106415, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7668711841106415 }, "534717141": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9558932781219482, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14369576234023818, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.04410672187805176, + "selected_utility_gain_over_base_at_8": -1.9558932781219482, "selector_regret_at_8": 1.9558932781219482, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9558932781219482 }, "551504635": { + "base_success": 0.0, + "base_utility": 0.7247228622436523, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2299325778712667, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7224790453910828, + "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7224790453910828, + "selected_utility_gain_over_base_at_8": -0.00224381685256958, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.2775209546089172 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2775209546089172, + "total_car_to_hidden_at_8": 1.2775209546089172 }, "565369392": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, "branch_car_at_8": 0.046260833740234375, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13991651305345584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.17366480827331543, + "proposal_oracle_utility_gain_over_base_at_8": -0.6261008381843567, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.12740397453308105, + "selected_utility_gain_over_base_at_8": -0.6723616719245911, "selector_regret_at_8": 0.046260833740234375, - "support_gap_at_8": 1.2529667715231578 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2529667715231578, + "total_car_to_hidden_at_8": 1.2992276052633922 }, "565621826": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "585904130": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, "branch_car_at_8": 0.012068324722349644, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07276320034061332, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032918572425842285, + "proposal_oracle_utility_gain_over_base_at_8": -0.04237352559963862, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02085024770349264, + "selected_utility_gain_over_base_at_8": -0.05444185032198826, "selector_regret_at_8": 0.012068324722349644, - "support_gap_at_8": 0.20472841958204904 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.20472841958204904, + "total_car_to_hidden_at_8": 0.2167967443043987 }, "626486813": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, "branch_car_at_8": 0.02500777377281338, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.2420589895144555, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07365635968744755, + "proposal_oracle_utility_gain_over_base_at_8": -0.43178365693893284, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.04864858591463417, + "selected_utility_gain_over_base_at_8": -0.4567914307117462, "selector_regret_at_8": 0.02500777377281338, - "support_gap_at_8": 0.5414478182792664 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5414478182792664, + "total_car_to_hidden_at_8": 0.5638198243686929 }, "636680137": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07628525991805817, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.013928472995758057, + "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013928472995758057, + "selected_utility_gain_over_base_at_8": -1.986071527004242, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.986071527004242 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.986071527004242, + "total_car_to_hidden_at_8": 1.986071527004242 }, "706479600": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.16443847051324711, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "750448296": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, "branch_car_at_8": 0.02701035887002945, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2344413298276417, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07342664152383804, + "proposal_oracle_utility_gain_over_base_at_8": -0.03810691833496094, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046416282653808594, + "selected_utility_gain_over_base_at_8": -0.06511727720499039, "selector_regret_at_8": 0.02701035887002945, - "support_gap_at_8": 0.16238094866275787 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.16238094866275787, + "total_car_to_hidden_at_8": 0.18939130753278732 }, "75070641": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.5863077491521835, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18559462278331035, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8128874897956848, + "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22657974064350128, + "selected_utility_gain_over_base_at_8": 0.02382810413837433, "selector_regret_at_8": 0.5863077491521835, - "support_gap_at_8": 1.1871125102043152 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1871125102043152, + "total_car_to_hidden_at_8": 1.7734202593564987 }, "767747424": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, "branch_car_at_8": 0.04561196081340313, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.07531396139335009, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11342880013398826, + "proposal_oracle_utility_gain_over_base_at_8": -0.4340953875798732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.06781683932058513, + "selected_utility_gain_over_base_at_8": -0.47970734839327633, "selector_regret_at_8": 0.04561196081340313, - "support_gap_at_8": 0.08526064571924508 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.08526064571924508, + "total_car_to_hidden_at_8": 0.12081881728954613 }, "77147264": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.2630803856693186, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.426636129617691, + "proposal_oracle_utility_gain_over_base_at_8": 0.005056649446487427, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.426636129617691, + "selected_utility_gain_over_base_at_8": 0.005056649446487427, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.573363870382309 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.573363870382309, + "total_car_to_hidden_at_8": 0.573363870382309 }, "794181626": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28339362113630795, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "957101489": { + "base_success": 0.0, + "base_utility": 0.20372186601161957, "branch_car_at_8": 1.7757539749145508, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07643021229779011, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22424602508544922, + "selected_utility_gain_over_base_at_8": 0.02052415907382965, "selector_regret_at_8": 1.7757539749145508, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7757539749145508 }, "998761162": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.9512543678283691, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28991947553571784, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04874563217163086, + "selected_utility_gain_over_base_at_8": -0.019070938229560852, "selector_regret_at_8": 1.9512543678283691, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9512543678283691 } } diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_task.json index 6df276a1051212dea711c3375052a95f2fe02a1e..bd263922de8de186c09351ff41888e14fea90c6e 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_task.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/metrics_by_task.json @@ -1,37 +1,117 @@ { "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.0670691976944606, "branch_car_at_8": 0.3250635440150897, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.21174314869082475, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.16666666666666666, + "proposal_oracle_utility_at_8": 1.3470948338508606, + "proposal_oracle_utility_gain_over_base_at_8": 0.2800256361564, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.0220312898357709, + "selected_utility_gain_over_base_at_8": -0.04503790785868963, "selector_regret_at_8": 0.3250635440150897, - "support_gap_at_8": 0.6529051661491394 + "success_selector_gap_at_8": 0.16666666666666666, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.6529051661491394, + "total_car_to_hidden_at_8": 0.977968710164229 }, "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.45726464316248894, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, "selector_regret_at_8": 0.45726464316248894, - "support_gap_at_8": 1.500250667333603 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.500250667333603, + "total_car_to_hidden_at_8": 1.9575153104960918 }, "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, "branch_car_at_8": 0.08196123232654132, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, "outcome_ptr_at_8": 0.29411764705882354, "pairwise_causal_calibration_ece": 0.14892514417161398, + "proposal_oracle_success_at_8": 0.11764705882352941, + "proposal_oracle_success_gain_over_base_at_8": -0.17647058823529413, + "proposal_oracle_utility_at_8": 0.3012250267818351, + "proposal_oracle_utility_gain_over_base_at_8": -0.35000489400151896, + "selected_success_at_8": 0.08823529411764706, + "selected_success_gain_over_base_at_8": -0.20588235294117646, + "selected_utility_at_8": 0.2192637944552938, + "selected_utility_gain_over_base_at_8": -0.4319661263280603, "selector_regret_at_8": 0.08196123232654132, - "support_gap_at_8": 0.4828804947901517 + "success_selector_gap_at_8": 0.029411764705882353, + "success_support_gap_at_8": 0.20588235294117646, + "success_total_car_to_hidden_at_8": 0.23529411764705882, + "support_gap_at_8": 0.4828804947901517, + "total_car_to_hidden_at_8": 0.5630580423440894 }, "PushCube-v1": { + "base_success": 0.38461538461538464, + "base_utility": 0.8223185676794785, "branch_car_at_8": 0.6069896691120588, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6153846153846154, "pairwise_causal_calibration_ece": 0.19653272589828236, + "proposal_oracle_success_at_8": 0.7692307692307693, + "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464, + "proposal_oracle_utility_at_8": 1.55294325030767, + "proposal_oracle_utility_gain_over_base_at_8": 0.7306246826281915, + "selected_success_at_8": 0.46153846153846156, + "selected_success_gain_over_base_at_8": 0.07692307692307693, + "selected_utility_at_8": 0.9459535811956112, + "selected_utility_gain_over_base_at_8": 0.12363501351613265, "selector_regret_at_8": 0.6069896691120588, - "support_gap_at_8": 0.44705674969233 + "success_selector_gap_at_8": 0.3076923076923077, + "success_support_gap_at_8": 0.23076923076923078, + "success_total_car_to_hidden_at_8": 0.5384615384615384, + "support_gap_at_8": 0.44705674969233, + "total_car_to_hidden_at_8": 1.0540464188043888 }, "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8036667982737223, "branch_car_at_8": 0.6179056455691655, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5333333333333333, "pairwise_causal_calibration_ece": 0.12910083675639442, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.13333333333333333, + "proposal_oracle_utility_at_8": 1.0861866732438406, + "proposal_oracle_utility_gain_over_base_at_8": 0.2825198749701182, + "selected_success_at_8": 0.13333333333333333, + "selected_success_gain_over_base_at_8": -0.06666666666666667, + "selected_utility_at_8": 0.468281027674675, + "selected_utility_gain_over_base_at_8": -0.33538577059904734, "selector_regret_at_8": 0.6179056455691655, - "support_gap_at_8": 0.9138133267561594 + "success_selector_gap_at_8": 0.2, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.8666666666666667, + "support_gap_at_8": 0.9138133267561594, + "total_car_to_hidden_at_8": 1.531718972325325 } } diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/report.md b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/report.md index d4bd1fa9706229f37779df37f188d662335df5f7..d2d30a701476b4b25b67a91023ed99d3975294cd 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/report.md +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/report.md @@ -4,8 +4,24 @@ K: `8` | Metric | N | Micro mean | 95% CI | Task macro | Seed macro | | --- | ---: | ---: | ---: | ---: | ---: | +| base_success | 69 | 0.2899 | [0.1884, 0.4203] | 0.2424 | 0.3298 | +| base_utility | 69 | 0.7455 | [0.5693, 0.9621] | 0.6995 | 0.8567 | | branch_car_at_8 | 69 | 0.3240 | [0.1877, 0.4805] | 0.4178 | 0.4633 | +| hidden_chart_oracle_success_at_8 | 69 | 0.6667 | [0.5507, 0.7826] | 0.8647 | 0.8358 | +| hidden_chart_oracle_utility_at_8 | 69 | 1.4000 | [1.1855, 1.6062] | 1.7565 | 1.7062 | | outcome_ptr_at_8 | 69 | 0.4348 | [0.3043, 0.5362] | 0.5886 | 0.4578 | | pairwise_causal_calibration_ece | 69 | 0.1583 | [0.1336, 0.1862] | 0.1567 | 0.1561 | +| proposal_oracle_success_at_8 | 69 | 0.3188 | [0.2174, 0.4348] | 0.3440 | 0.4326 | +| proposal_oracle_success_gain_over_base_at_8 | 69 | 0.0290 | [-0.0870, 0.1449] | 0.1016 | 0.1028 | +| proposal_oracle_utility_at_8 | 69 | 0.8015 | [0.6154, 1.0082] | 0.9574 | 1.0621 | +| proposal_oracle_utility_gain_over_base_at_8 | 69 | 0.0560 | [-0.1555, 0.2702] | 0.2579 | 0.2055 | +| selected_success_at_8 | 69 | 0.1884 | [0.1159, 0.2899] | 0.2033 | 0.2411 | +| selected_success_gain_over_base_at_8 | 69 | -0.1014 | [-0.2174, 0.0000] | -0.0391 | -0.0887 | +| selected_utility_at_8 | 69 | 0.4776 | [0.3230, 0.6844] | 0.5396 | 0.5988 | +| selected_utility_gain_over_base_at_8 | 69 | -0.2680 | [-0.4864, -0.0695] | -0.1599 | -0.2579 | | selector_regret_at_8 | 69 | 0.3240 | [0.1877, 0.4805] | 0.4178 | 0.4633 | +| success_selector_gap_at_8 | 69 | 0.1304 | [0.0580, 0.2174] | 0.1408 | 0.1915 | +| success_support_gap_at_8 | 69 | 0.3478 | [0.2319, 0.4638] | 0.5207 | 0.4032 | +| success_total_car_to_hidden_at_8 | 69 | 0.4783 | [0.3623, 0.5942] | 0.6614 | 0.5947 | | support_gap_at_8 | 69 | 0.5993 | [0.4135, 0.7763] | 0.7994 | 0.6444 | +| total_car_to_hidden_at_8 | 69 | 0.9224 | [0.7123, 1.1263] | 1.2169 | 1.1074 | diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/table.tex b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/table.tex index 10047a60eeb8718f3daaff6ccb4238591086324b..a288b0c7e4867a5949452f6ad9557dfb8b82cb0f 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/table.tex +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/measured_metrics/table.tex @@ -3,10 +3,26 @@ \toprule Metric & N & Micro mean & CI low & CI high \\ \midrule +base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\ +base\_utility & 69 & 0.7455 & 0.5693 & 0.9621 \\ branch\_car\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\ +hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\ outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3043 & 0.5362 \\ pairwise\_causal\_calibration\_ece & 69 & 0.1583 & 0.1336 & 0.1862 \\ +proposal\_oracle\_success\_at\_8 & 69 & 0.3188 & 0.2174 & 0.4348 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.0290 & -0.0870 & 0.1449 \\ +proposal\_oracle\_utility\_at\_8 & 69 & 0.8015 & 0.6154 & 1.0082 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.0560 & -0.1555 & 0.2702 \\ +selected\_success\_at\_8 & 69 & 0.1884 & 0.1159 & 0.2899 \\ +selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.1014 & -0.2174 & 0.0000 \\ +selected\_utility\_at\_8 & 69 & 0.4776 & 0.3230 & 0.6844 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.2680 & -0.4864 & -0.0695 \\ selector\_regret\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\ +success\_selector\_gap\_at\_8 & 69 & 0.1304 & 0.0580 & 0.2174 \\ +success\_support\_gap\_at\_8 & 69 & 0.3478 & 0.2319 & 0.4638 \\ +success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4783 & 0.3623 & 0.5942 \\ support\_gap\_at\_8 & 69 & 0.5993 & 0.4135 & 0.7763 \\ +total\_car\_to\_hidden\_at\_8 & 69 & 0.9224 & 0.7123 & 1.1263 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/metrics.json b/workspace/runs/ctt_residual_rollout_val69_seed1/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..57465260b451f21cc88780ca01107422d61facbe --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/metrics.json @@ -0,0 +1,1918 @@ +{ + "checkpoint": "runs/ctt_residual_full_seed1/model.pt", + "k": 8, + "measured_metric_summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4521276595744681, + "low": 0.20921985815602834, + "mean": 0.3297872340425532, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3297872340425532, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.33815987933634994, + "low": 0.10666666666666666, + "mean": 0.24241327300150828, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24241327300150828, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.42028985507246375, + "low": 0.18840579710144928, + "mean": 0.2898550724637681, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0927389474422176, + "low": 0.6324926499567979, + "mean": 0.8566858090574239, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8566858090574239, + "per_seed": { + "1025740212": 0.528373658657074, + "1055157354": 0.17649109661579132, + "1055567618": 2.0, + "1113268940": 0.42231252789497375, + "1113371261": 2.0, + "1176943004": 0.058033183217048645, + "1189392917": 0.10841864347457886, + "1231780997": 0.7270061373710632, + "1271718007": 1.415381133556366, + "1273247072": 0.15342965722084045, + "1358421325": 0.8324764966964722, + "1375492786": 2.0, + "141984615": 0.09381213908394177, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.19844311475753784, + "1528740417": 2.0, + "1549324805": 1.3787474483251572, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.1463065892457962, + "1881223807": 0.0320269912481308, + "1919198188": 0.03719887137413025, + "1991414747": 2.0, + "1997090716": 0.13927479088306427, + "2015956957": 0.06711978986859321, + "2054899126": 2.0, + "211245842": 0.8388344049453735, + "2141802816": 0.07080724835395813, + "256770874": 0.8557480573654175, + "278008524": 0.10747186839580536, + "416183106": 0.20032641291618347, + "534717141": 2.0, + "551504635": 0.7247228622436523, + "565369392": 0.7997656464576721, + "565621826": 2.0, + "585904130": 0.07529209802548091, + "626486813": 0.5054400166263804, + "636680137": 2.0, + "706479600": 0.8350887894630432, + "750448296": 0.11153355985879898, + "75070641": 0.20275163650512695, + "767747424": 0.5475241877138615, + "77147264": 1.4215794801712036, + "794181626": 0.18095605075359344, + "957101489": 0.20372186601161957, + "998761162": 0.06781657040119171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9312208624280917, + "low": 0.4135245136419933, + "mean": 0.6995428283303713, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6995428283303713, + "per_task": { + "LiftPegUpright-v1": 1.0670691976944606, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.8223185676794785, + "StackCube-v1": 0.8036667982737223 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9620560415399572, + "low": 0.5693137269615587, + "mean": 0.7455479058545029, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "branch_car_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6905135366465427, + "low": 0.2873830998237971, + "mean": 0.46333538608845154, + "n": 47, + "num_samples": 500 + }, + "mean": 0.46333538608845154, + "per_seed": { + "1025740212": 0.47739143669605255, + "1055157354": 0.6370042562484741, + "1055567618": 1.8253512382507324, + "1113268940": 0.18306860327720642, + "1113371261": 0.0, + "1176943004": 1.974869817495346, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.00353144109249115, + "1273247072": 0.45726464316248894, + "1358421325": 0.6467066705226898, + "1375492786": 0.00047226750757545233, + "141984615": 0.05730179324746132, + "1475440973": 1.289921224117279, + "1483645336": 1.9307887554168701, + "1522272686": 0.03616611659526825, + "1528740417": 0.0, + "1549324805": 0.0015795694974561532, + "1607773860": 0.6036040186882019, + "1665739511": 0.0, + "1747178030": 0.027157030999660492, + "1881223807": 0.009028241038322449, + "1919198188": 0.0, + "1991414747": 1.9863876849412918, + "1997090716": 0.0, + "2015956957": 0.022195855714380743, + "2054899126": 0.0, + "211245842": 0.6792904287576675, + "2141802816": 0.019766509532928467, + "256770874": 0.7044661641120911, + "278008524": 0.011409573198761791, + "416183106": 1.7668711841106415, + "534717141": 1.9558932781219482, + "551504635": 0.0, + "565369392": 0.046260833740234375, + "565621826": 0.0, + "585904130": 0.012068324722349644, + "626486813": 0.02500777377281338, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.02701035887002945, + "75070641": 0.5863077491521835, + "767747424": 0.04561196081340313, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.7757539749145508, + "998761162": 1.9512543678283691 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5835942497964088, + "low": 0.22782261933967035, + "mean": 0.4178369468370689, + "n": 5, + "num_samples": 500 + }, + "mean": 0.4178369468370689, + "per_task": { + "LiftPegUpright-v1": 0.3250635440150897, + "PickCube-v1": 0.45726464316248894, + "PullCube-v1": 0.08196123232654132, + "PushCube-v1": 0.6069896691120588, + "StackCube-v1": 0.6179056455691655 + } + }, + "metric": "branch_car_at_8", + "micro": { + "confidence": 0.95, + "high": 0.48046939117341314, + "low": 0.18767418079685583, + "mean": 0.32396779983115476, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.922695035460993, + "low": 0.7351063829787235, + "mean": 0.8358156028368794, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8358156028368794, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.782608695652174, + "low": 0.5507246376811594, + "mean": 0.6666666666666666, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8615907955016442, + "low": 1.529367521088174, + "mean": 1.7062028111638965, + "n": 47, + "num_samples": 500 + }, + "mean": 1.7062028111638965, + "per_seed": { + "1025740212": 2.0, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 2.0, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1358421325": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.20044784247875214, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 2.0, + "1528740417": 2.0, + "1549324805": 1.4027505218982697, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 2.0, + "1881223807": 2.0, + "1919198188": 2.0, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.5298813670873642, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 2.0, + "278008524": 1.113811194896698, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 1.4266315797964733, + "565621826": 2.0, + "585904130": 0.23764699200789133, + "626486813": 0.6124684102833271, + "636680137": 2.0, + "706479600": 2.0, + "750448296": 0.23580759018659592, + "75070641": 2.0, + "767747424": 0.18863565661013126, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.6062410234109215, + "low": 1.1855228276572365, + "mean": 1.3999846732054932, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "outcome_ptr_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5886524822695036, + "low": 0.3191489361702128, + "mean": 0.4578014184397163, + "n": 47, + "num_samples": 500 + }, + "mean": 0.4578014184397163, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.6666666666666666, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.6, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.5, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.75, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8133333333333332, + "low": 0.4062141779788838, + "mean": 0.5885671191553545, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5885671191553545, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 1.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.6153846153846154, + "StackCube-v1": 0.5333333333333333 + } + }, + "metric": "outcome_ptr_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5362318840579711, + "low": 0.30434782608695654, + "mean": 0.43478260869565216, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.18010258206023347, + "low": 0.13094723444140935, + "mean": 0.15606796759963956, + "n": 47, + "num_samples": 500 + }, + "mean": 0.15606796759963956, + "per_seed": { + "1025740212": 0.12992986623548633, + "1055157354": 0.07450750352404123, + "1055567618": 0.318234334809661, + "1113268940": 0.10725283874677279, + "1113371261": 0.04977883974023356, + "1176943004": 0.1815411965846294, + "1189392917": 0.2180235973132697, + "1231780997": 0.28322114525972475, + "1271718007": 0.07023931523066618, + "1273247072": 0.096994407338536, + "1358421325": 0.046694794435454434, + "1375492786": 0.14027669602920534, + "141984615": 0.08533231968443573, + "1475440973": 0.2443552809335613, + "1483645336": 0.13752192684190806, + "1522272686": 0.28555947838938045, + "1528740417": 0.2757671830981365, + "1549324805": 0.28625156462411294, + "1607773860": 0.18975917078230178, + "1665739511": 0.0, + "1747178030": 0.3089551740932353, + "1881223807": 0.13674585276395504, + "1919198188": 0.13114443223224567, + "1991414747": 0.1306768270010748, + "1997090716": 0.2554442550173647, + "2015956957": 0.16317219295797403, + "2054899126": 0.0, + "211245842": 0.07548939908426067, + "2141802816": 0.07786951284417709, + "256770874": 0.03135306008748828, + "278008524": 0.24605586147071373, + "416183106": 0.07978206783428046, + "534717141": 0.14369576234023818, + "551504635": 0.2299325778712667, + "565369392": 0.13991651305345584, + "565621826": 0.0, + "585904130": 0.07276320034061332, + "626486813": 0.2420589895144555, + "636680137": 0.07628525991805817, + "706479600": 0.16443847051324711, + "750448296": 0.2344413298276417, + "75070641": 0.18559462278331035, + "767747424": 0.07531396139335009, + "77147264": 0.2630803856693186, + "794181626": 0.28339362113630795, + "957101489": 0.07643021229779011, + "998761162": 0.28991947553571784 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.19005329411145716, + "low": 0.12022312647229497, + "mean": 0.15665925257113028, + "n": 5, + "num_samples": 500 + }, + "mean": 0.15665925257113028, + "per_task": { + "LiftPegUpright-v1": 0.21174314869082475, + "PickCube-v1": 0.096994407338536, + "PullCube-v1": 0.14892514417161398, + "PushCube-v1": 0.19653272589828236, + "StackCube-v1": 0.12910083675639442 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.186245353107692, + "low": 0.13360163870735103, + "mean": 0.15829487230930359, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5673758865248226, + "low": 0.2943262411347518, + "mean": 0.4326241134751774, + "n": 47, + "num_samples": 500 + }, + "mean": 0.4326241134751774, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5850678733031673, + "low": 0.1, + "mean": 0.3440422322775264, + "n": 5, + "num_samples": 500 + }, + "mean": 0.3440422322775264, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 0.0, + "PullCube-v1": 0.11764705882352941, + "PushCube-v1": 0.7692307692307693, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "proposal_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.43478260869565216, + "low": 0.21739130434782608, + "mean": 0.3188405797101449, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.25709219858156024, + "low": -0.03368794326241135, + "mean": 0.10283687943262412, + "n": 47, + "num_samples": 500 + }, + "mean": 0.10283687943262412, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": -0.3333333333333333, + "1607773860": -1.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2641025641025641, + "low": -0.07058823529411765, + "mean": 0.10162895927601809, + "n": 5, + "num_samples": 500 + }, + "mean": 0.10162895927601809, + "per_task": { + "LiftPegUpright-v1": 0.16666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": -0.17647058823529413, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.13333333333333333 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.14492753623188406, + "low": -0.08695652173913043, + "mean": 0.028985507246376812, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.318000317976896, + "low": 0.8185871909540939, + "mean": 1.0621498337636393, + "n": 47, + "num_samples": 500 + }, + "mean": 1.0621498337636393, + "per_seed": { + "1025740212": 0.6983401775360107, + "1055157354": 0.8232162594795227, + "1055567618": 2.0, + "1113268940": 0.6617497801780701, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 1.1110476776957512, + "1273247072": 0.4997493326663971, + "1358421325": 0.8275434374809265, + "1375492786": 0.0017900466918945312, + "141984615": 0.09240151196718216, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.23248937726020813, + "1528740417": 2.0, + "1549324805": 0.6785067518552145, + "1607773860": 0.8267073631286621, + "1665739511": 2.0, + "1747178030": 0.11134227365255356, + "1881223807": 0.034886181354522705, + "1919198188": 0.046244338154792786, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.05629209652543068, + "2054899126": 2.0, + "211245842": 0.8358442187309265, + "2141802816": 0.09589323401451111, + "256770874": 0.8125, + "278008524": 0.03147180867381394, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 0.7224790453910828, + "565369392": 0.17366480827331543, + "565621826": 2.0, + "585904130": 0.032918572425842285, + "626486813": 0.07365635968744755, + "636680137": 0.013928472995758057, + "706479600": 2.0, + "750448296": 0.07342664152383804, + "75070641": 0.8128874897956848, + "767747424": 0.11342880013398826, + "77147264": 1.426636129617691, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3423044667794155, + "low": 0.537627078428061, + "mean": 0.9574398233701207, + "n": 5, + "num_samples": 500 + }, + "mean": 0.9574398233701207, + "per_task": { + "LiftPegUpright-v1": 1.3470948338508606, + "PickCube-v1": 0.4997493326663971, + "PullCube-v1": 0.3012250267818351, + "PushCube-v1": 1.55294325030767, + "StackCube-v1": 1.0861866732438406 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.0082228477517872, + "low": 0.6154177648528223, + "mean": 0.8015221970871199, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4860803771314892, + "low": -0.031156662437926107, + "mean": 0.20546402470621525, + "n": 47, + "num_samples": 500 + }, + "mean": 0.20546402470621525, + "per_seed": { + "1025740212": 0.16996651887893677, + "1055157354": 0.6467251628637314, + "1055567618": 0.0, + "1113268940": 0.2394372522830963, + "1113371261": 0.0, + "1176943004": 1.9419668167829514, + "1189392917": 1.8915813565254211, + "1231780997": 1.2729938626289368, + "1271718007": -0.3043334558606148, + "1273247072": 0.34631967544555664, + "1358421325": -0.004933059215545654, + "1375492786": -1.9982099533081055, + "141984615": -0.001410627116759618, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.03404626250267029, + "1528740417": 0.0, + "1549324805": -0.7002406964699427, + "1607773860": -1.173292636871338, + "1665739511": 0.0, + "1747178030": -0.034964315593242645, + "1881223807": 0.0028591901063919067, + "1919198188": 0.009045466780662537, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": -0.010827693343162536, + "2054899126": 0.0, + "211245842": -0.0029901862144470215, + "2141802816": 0.02508598566055298, + "256770874": -0.04324805736541748, + "278008524": -0.07600005972199142, + "416183106": 1.7996735870838165, + "534717141": 0.0, + "551504635": -0.00224381685256958, + "565369392": -0.6261008381843567, + "565621826": 0.0, + "585904130": -0.04237352559963862, + "626486813": -0.43178365693893284, + "636680137": -1.986071527004242, + "706479600": 1.1649112105369568, + "750448296": -0.03810691833496094, + "75070641": 0.6101358532905579, + "767747424": -0.4340953875798732, + "77147264": 0.005056649446487427, + "794181626": 1.8190439492464066, + "957101489": 1.7962781339883804, + "998761162": 1.9321834295988083 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5508839118022185, + "low": -0.08473387408052005, + "mean": 0.2578969950397495, + "n": 5, + "num_samples": 500 + }, + "mean": 0.2578969950397495, + "per_task": { + "LiftPegUpright-v1": 0.2800256361564, + "PickCube-v1": 0.34631967544555664, + "PullCube-v1": -0.35000489400151896, + "PushCube-v1": 0.7306246826281915, + "StackCube-v1": 0.2825198749701182 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2701519333851942, + "low": -0.15554412837018786, + "mean": 0.055974291232617004, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3475177304964539, + "low": 0.1347517730496454, + "mean": 0.24113475177304963, + "n": 47, + "num_samples": 500 + }, + "mean": 0.24113475177304963, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3589743589743589, + "low": 0.05333333333333333, + "mean": 0.20328808446455504, + "n": 5, + "num_samples": 500 + }, + "mean": 0.20328808446455504, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.08823529411764706, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.13333333333333333 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2898550724637681, + "low": 0.11594202898550725, + "mean": 0.18840579710144928, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.04609929078014185, + "low": -0.23404255319148937, + "mean": -0.08865248226950355, + "n": 47, + "num_samples": 500 + }, + "mean": -0.08865248226950355, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": -1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": -1.0, + "1483645336": -1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": -0.3333333333333333, + "1607773860": -1.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": -1.0, + "1997090716": 1.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": -1.0, + "551504635": 0.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.03282051282051283, + "low": -0.13686274509803922, + "mean": -0.03912518853695324, + "n": 5, + "num_samples": 500 + }, + "mean": -0.03912518853695324, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": -0.20588235294117646, + "PushCube-v1": 0.07692307692307693, + "StackCube-v1": -0.06666666666666667 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.0, + "low": -0.21739130434782608, + "mean": -0.10144927536231885, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7957249261758369, + "low": 0.38785451346428745, + "mean": 0.5988144476751877, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5988144476751877, + "per_seed": { + "1025740212": 0.2209487408399582, + "1055157354": 0.18621200323104858, + "1055567618": 0.17464876174926758, + "1113268940": 0.47868117690086365, + "1113371261": 2.0, + "1176943004": 0.02513018250465393, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 1.10751623660326, + "1273247072": 0.04248468950390816, + "1358421325": 0.1808367669582367, + "1375492786": 0.001317779184319079, + "141984615": 0.03509971871972084, + "1475440973": 0.710078775882721, + "1483645336": 0.06921124458312988, + "1522272686": 0.19632326066493988, + "1528740417": 2.0, + "1549324805": 0.6769271823577583, + "1607773860": 0.2231033444404602, + "1665739511": 2.0, + "1747178030": 0.08418524265289307, + "1881223807": 0.025857940316200256, + "1919198188": 0.046244338154792786, + "1991414747": 0.013612315058708191, + "1997090716": 2.0, + "2015956957": 0.034096240811049935, + "2054899126": 2.0, + "211245842": 0.15655378997325897, + "2141802816": 0.07612672448158264, + "256770874": 0.10803383588790894, + "278008524": 0.020062235475052148, + "416183106": 0.23312881588935852, + "534717141": 0.04410672187805176, + "551504635": 0.7224790453910828, + "565369392": 0.12740397453308105, + "565621826": 2.0, + "585904130": 0.02085024770349264, + "626486813": 0.04864858591463417, + "636680137": 0.013928472995758057, + "706479600": 2.0, + "750448296": 0.046416282653808594, + "75070641": 0.22657974064350128, + "767747424": 0.06781683932058513, + "77147264": 1.426636129617691, + "794181626": 2.0, + "957101489": 0.22424602508544922, + "998761162": 0.04874563217163086 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8462622490316436, + "low": 0.19835559911861575, + "mean": 0.5396028765330518, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5396028765330518, + "per_task": { + "LiftPegUpright-v1": 1.0220312898357709, + "PickCube-v1": 0.04248468950390816, + "PullCube-v1": 0.2192637944552938, + "PushCube-v1": 0.9459535811956112, + "StackCube-v1": 0.468281027674675 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.6843658976587534, + "low": 0.3229842852494018, + "mean": 0.4775543972559651, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": -0.010564623637940616, + "low": -0.5054863751322652, + "mean": -0.25787136138223626, + "n": 47, + "num_samples": 500 + }, + "mean": -0.25787136138223626, + "per_seed": { + "1025740212": -0.3074249178171158, + "1055157354": 0.009720906615257263, + "1055567618": -1.8253512382507324, + "1113268940": 0.05636864900588989, + "1113371261": 0.0, + "1176943004": -0.032903000712394714, + "1189392917": 1.8915813565254211, + "1231780997": 1.2729938626289368, + "1271718007": -0.3078648969531059, + "1273247072": -0.1109449677169323, + "1358421325": -0.6516397297382355, + "1375492786": -1.998682220815681, + "141984615": -0.05871242036422094, + "1475440973": -1.289921224117279, + "1483645336": -1.9307887554168701, + "1522272686": -0.0021198540925979614, + "1528740417": 0.0, + "1549324805": -0.7018202659673989, + "1607773860": -1.7768966555595398, + "1665739511": 0.0, + "1747178030": -0.06212134659290314, + "1881223807": -0.006169050931930542, + "1919198188": 0.009045466780662537, + "1991414747": -1.9863876849412918, + "1997090716": 1.8607252091169357, + "2015956957": -0.033023549057543275, + "2054899126": 0.0, + "211245842": -0.6822806149721146, + "2141802816": 0.005319476127624512, + "256770874": -0.7477142214775085, + "278008524": -0.08740963292075321, + "416183106": 0.03280240297317505, + "534717141": -1.9558932781219482, + "551504635": -0.00224381685256958, + "565369392": -0.6723616719245911, + "565621826": 0.0, + "585904130": -0.05444185032198826, + "626486813": -0.4567914307117462, + "636680137": -1.986071527004242, + "706479600": 1.1649112105369568, + "750448296": -0.06511727720499039, + "75070641": 0.02382810413837433, + "767747424": -0.47970734839327633, + "77147264": 0.005056649446487427, + "794181626": 1.8190439492464066, + "957101489": 0.02052415907382965, + "998761162": -0.019070938229560852 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": -0.0019037275818678035, + "low": -0.33526441148838354, + "mean": -0.1599399517973194, + "n": 5, + "num_samples": 500 + }, + "mean": -0.1599399517973194, + "per_task": { + "LiftPegUpright-v1": -0.04503790785868963, + "PickCube-v1": -0.1109449677169323, + "PullCube-v1": -0.4319661263280603, + "PushCube-v1": 0.12363501351613265, + "StackCube-v1": -0.33538577059904734 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": -0.06954942767143897, + "low": -0.4864171527484702, + "mean": -0.26799350859853777, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6905135366465427, + "low": 0.2873830998237971, + "mean": 0.46333538608845154, + "n": 47, + "num_samples": 500 + }, + "mean": 0.46333538608845154, + "per_seed": { + "1025740212": 0.47739143669605255, + "1055157354": 0.6370042562484741, + "1055567618": 1.8253512382507324, + "1113268940": 0.18306860327720642, + "1113371261": 0.0, + "1176943004": 1.974869817495346, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.00353144109249115, + "1273247072": 0.45726464316248894, + "1358421325": 0.6467066705226898, + "1375492786": 0.00047226750757545233, + "141984615": 0.05730179324746132, + "1475440973": 1.289921224117279, + "1483645336": 1.9307887554168701, + "1522272686": 0.03616611659526825, + "1528740417": 0.0, + "1549324805": 0.0015795694974561532, + "1607773860": 0.6036040186882019, + "1665739511": 0.0, + "1747178030": 0.027157030999660492, + "1881223807": 0.009028241038322449, + "1919198188": 0.0, + "1991414747": 1.9863876849412918, + "1997090716": 0.0, + "2015956957": 0.022195855714380743, + "2054899126": 0.0, + "211245842": 0.6792904287576675, + "2141802816": 0.019766509532928467, + "256770874": 0.7044661641120911, + "278008524": 0.011409573198761791, + "416183106": 1.7668711841106415, + "534717141": 1.9558932781219482, + "551504635": 0.0, + "565369392": 0.046260833740234375, + "565621826": 0.0, + "585904130": 0.012068324722349644, + "626486813": 0.02500777377281338, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.02701035887002945, + "75070641": 0.5863077491521835, + "767747424": 0.04561196081340313, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.7757539749145508, + "998761162": 1.9512543678283691 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5835942497964088, + "low": 0.22782261933967035, + "mean": 0.4178369468370689, + "n": 5, + "num_samples": 500 + }, + "mean": 0.4178369468370689, + "per_task": { + "LiftPegUpright-v1": 0.3250635440150897, + "PickCube-v1": 0.45726464316248894, + "PullCube-v1": 0.08196123232654132, + "PushCube-v1": 0.6069896691120588, + "StackCube-v1": 0.6179056455691655 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.48046939117341314, + "low": 0.18767418079685583, + "mean": 0.32396779983115476, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3191489361702128, + "low": 0.0851063829787234, + "mean": 0.19148936170212766, + "n": 47, + "num_samples": 500 + }, + "mean": 0.19148936170212766, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.24307692307692305, + "low": 0.045098039215686274, + "mean": 0.14075414781297133, + "n": 5, + "num_samples": 500 + }, + "mean": 0.14075414781297133, + "per_task": { + "LiftPegUpright-v1": 0.16666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.029411764705882353, + "PushCube-v1": 0.3076923076923077, + "StackCube-v1": 0.2 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.21739130434782608, + "low": 0.057971014492753624, + "mean": 0.13043478260869565, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.551418439716312, + "low": 0.2726950354609929, + "mean": 0.4031914893617021, + "n": 47, + "num_samples": 500 + }, + "mean": 0.4031914893617021, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7794871794871795, + "low": 0.2746606334841629, + "mean": 0.5206636500754148, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5206636500754148, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 1.0, + "PullCube-v1": 0.20588235294117646, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.463768115942029, + "low": 0.2318840579710145, + "mean": 0.34782608695652173, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7304964539007093, + "low": 0.45886524822695035, + "mean": 0.5946808510638298, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5946808510638298, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8810256410256411, + "low": 0.41719457013574657, + "mean": 0.6614177978883861, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6614177978883861, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.23529411764705882, + "PushCube-v1": 0.5384615384615384, + "StackCube-v1": 0.8666666666666667 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5942028985507246, + "low": 0.36231884057971014, + "mean": 0.4782608695652174, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8704533781212297, + "low": 0.4250871043820726, + "mean": 0.6443930889275058, + "n": 47, + "num_samples": 500 + }, + "mean": 0.6443930889275058, + "per_seed": { + "1025740212": 1.3016598224639893, + "1055157354": 1.1767837405204773, + "1055567618": 0.0, + "1113268940": 1.33825021982193, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.8889523223042488, + "1273247072": 1.500250667333603, + "1358421325": 1.1724565625190735, + "1375492786": 0.2416609227657318, + "141984615": 0.11134201536575954, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.7675106227397919, + "1528740417": 0.0, + "1549324805": 0.7242437700430552, + "1607773860": 1.173292636871338, + "1665739511": 0.0, + "1747178030": 1.8886577263474464, + "1881223807": 1.9651138186454773, + "1919198188": 1.9537556618452072, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.4735892705619335, + "2054899126": 0.0, + "211245842": 1.1641557812690735, + "2141802816": 1.904106765985489, + "256770874": 1.1875, + "278008524": 1.082339386222884, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 1.2775209546089172, + "565369392": 1.2529667715231578, + "565621826": 0.0, + "585904130": 0.20472841958204904, + "626486813": 0.5414478182792664, + "636680137": 1.986071527004242, + "706479600": 0.0, + "750448296": 0.16238094866275787, + "75070641": 1.1871125102043152, + "767747424": 0.08526064571924508, + "77147264": 0.573363870382309, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.1794891647094239, + "low": 0.5025559310228205, + "mean": 0.7993812809442767, + "n": 5, + "num_samples": 500 + }, + "mean": 0.7993812809442767, + "per_task": { + "LiftPegUpright-v1": 0.6529051661491394, + "PickCube-v1": 1.500250667333603, + "PullCube-v1": 0.4828804947901517, + "PushCube-v1": 0.44705674969233, + "StackCube-v1": 0.9138133267561594 + } + }, + "metric": "support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.7763315162122033, + "low": 0.4135484881631598, + "mean": 0.5993413932526998, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.339665490579098, + "low": 0.8604674879511063, + "mean": 1.1073883634887087, + "n": 47, + "num_samples": 500 + }, + "mean": 1.1073883634887087, + "per_seed": { + "1025740212": 1.7790512591600418, + "1055157354": 1.8137879967689514, + "1055567618": 1.8253512382507324, + "1113268940": 1.5213188230991364, + "1113371261": 0.0, + "1176943004": 1.974869817495346, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.89248376339674, + "1273247072": 1.9575153104960918, + "1358421325": 1.8191632330417633, + "1375492786": 0.24213319027330726, + "141984615": 0.1653481237590313, + "1475440973": 1.289921224117279, + "1483645336": 1.9307887554168701, + "1522272686": 1.8036767393350601, + "1528740417": 0.0, + "1549324805": 0.7258233395405114, + "1607773860": 1.7768966555595398, + "1665739511": 0.0, + "1747178030": 1.915814757347107, + "1881223807": 1.9741420596837997, + "1919198188": 1.9537556618452072, + "1991414747": 1.9863876849412918, + "1997090716": 0.0, + "2015956957": 0.49578512627631427, + "2054899126": 0.0, + "211245842": 1.843446210026741, + "2141802816": 1.9238732755184174, + "256770874": 1.891966164112091, + "278008524": 1.0937489594216458, + "416183106": 1.7668711841106415, + "534717141": 1.9558932781219482, + "551504635": 1.2775209546089172, + "565369392": 1.2992276052633922, + "565621826": 0.0, + "585904130": 0.2167967443043987, + "626486813": 0.5638198243686929, + "636680137": 1.986071527004242, + "706479600": 0.0, + "750448296": 0.18939130753278732, + "75070641": 1.7734202593564987, + "767747424": 0.12081881728954613, + "77147264": 0.573363870382309, + "794181626": 0.0, + "957101489": 1.7757539749145508, + "998761162": 1.9512543678283691 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.676446722795566, + "low": 0.8272199847642051, + "mean": 1.2168614908268247, + "n": 5, + "num_samples": 500 + }, + "mean": 1.2168614908268247, + "per_task": { + "LiftPegUpright-v1": 0.977968710164229, + "PickCube-v1": 1.9575153104960918, + "PullCube-v1": 0.5630580423440894, + "PushCube-v1": 1.0540464188043888, + "StackCube-v1": 1.531718972325325 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 1.1262573117906551, + "low": 0.7123256691755133, + "mean": 0.9224302759495281, + "n": 69, + "num_samples": 500 + }, + "n": 69 + } + }, + "num_rows": 69, + "report_type": "ctt_generated_measured_rollout", + "schema_version": 1, + "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957", + "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "success_summary": { + "base_success_rate": 0.2898550724637681, + "base_utility_mean": 0.7455479058545029, + "hidden_chart_oracle_success_rate": 0.6666666666666666, + "hidden_chart_oracle_utility_mean": 1.3999846732054932, + "max_restore_error": 1.4901161193847656e-08, + "proposal_oracle_success_gain_over_base": 0.028985507246376812, + "proposal_oracle_success_rate": 0.3188405797101449, + "proposal_oracle_utility_mean": 0.8015221970871199, + "selected_success_gain_over_base": -0.10144927536231885, + "selected_success_rate": 0.18840579710144928, + "selected_utility_mean": 0.4775543972559651, + "success_selector_gap": 0.13043478260869565, + "success_support_gap": 0.34782608695652173 + }, + "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42", + "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381" +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..c8db434f785c6bcd93354109ac6e9a99edcd55da --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_seed.json @@ -0,0 +1,1083 @@ +{ + "1025740212": { + "base_success": 0.0, + "base_utility": 0.528373658657074, + "branch_car_at_8": 0.47739143669605255, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.12992986623548633, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6983401775360107, + "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2209487408399582, + "selected_utility_gain_over_base_at_8": -0.3074249178171158, + "selector_regret_at_8": 0.47739143669605255, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3016598224639893, + "total_car_to_hidden_at_8": 1.7790512591600418 + }, + "1055157354": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, + "branch_car_at_8": 0.6370042562484741, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07450750352404123, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8232162594795227, + "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18621200323104858, + "selected_utility_gain_over_base_at_8": 0.009720906615257263, + "selector_regret_at_8": 0.6370042562484741, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1767837405204773, + "total_car_to_hidden_at_8": 1.8137879967689514 + }, + "1055567618": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.8253512382507324, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.318234334809661, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.17464876174926758, + "selected_utility_gain_over_base_at_8": -1.8253512382507324, + "selector_regret_at_8": 1.8253512382507324, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8253512382507324 + }, + "1113268940": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, + "branch_car_at_8": 0.18306860327720642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10725283874677279, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6617497801780701, + "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.47868117690086365, + "selected_utility_gain_over_base_at_8": 0.05636864900588989, + "selector_regret_at_8": 0.18306860327720642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.33825021982193, + "total_car_to_hidden_at_8": 1.5213188230991364 + }, + "1113371261": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.04977883974023356, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1176943004": { + "base_success": 0.0, + "base_utility": 0.058033183217048645, + "branch_car_at_8": 1.974869817495346, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1815411965846294, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02513018250465393, + "selected_utility_gain_over_base_at_8": -0.032903000712394714, + "selector_regret_at_8": 1.974869817495346, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.974869817495346 + }, + "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2180235973132697, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1231780997": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28322114525972475, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.2729938626289368, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1271718007": { + "base_success": 0.5, + "base_utility": 1.415381133556366, + "branch_car_at_8": 0.00353144109249115, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07023931523066618, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.1110476776957512, + "proposal_oracle_utility_gain_over_base_at_8": -0.3043334558606148, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.10751623660326, + "selected_utility_gain_over_base_at_8": -0.3078648969531059, + "selector_regret_at_8": 0.00353144109249115, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.8889523223042488, + "total_car_to_hidden_at_8": 0.89248376339674 + }, + "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.45726464316248894, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, + "selector_regret_at_8": 0.45726464316248894, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.500250667333603, + "total_car_to_hidden_at_8": 1.9575153104960918 + }, + "1358421325": { + "base_success": 0.0, + "base_utility": 0.8324764966964722, + "branch_car_at_8": 0.6467066705226898, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.046694794435454434, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8275434374809265, + "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1808367669582367, + "selected_utility_gain_over_base_at_8": -0.6516397297382355, + "selector_regret_at_8": 0.6467066705226898, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1724565625190735, + "total_car_to_hidden_at_8": 1.8191632330417633 + }, + "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.00047226750757545233, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.14027669602920534, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0017900466918945312, + "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001317779184319079, + "selected_utility_gain_over_base_at_8": -1.998682220815681, + "selector_regret_at_8": 0.00047226750757545233, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.2416609227657318, + "total_car_to_hidden_at_8": 0.24213319027330726 + }, + "141984615": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, + "branch_car_at_8": 0.05730179324746132, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.08533231968443573, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09240151196718216, + "proposal_oracle_utility_gain_over_base_at_8": -0.001410627116759618, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03509971871972084, + "selected_utility_gain_over_base_at_8": -0.05871242036422094, + "selector_regret_at_8": 0.05730179324746132, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.11134201536575954, + "total_car_to_hidden_at_8": 0.1653481237590313 + }, + "1475440973": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.289921224117279, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2443552809335613, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.710078775882721, + "selected_utility_gain_over_base_at_8": -1.289921224117279, + "selector_regret_at_8": 1.289921224117279, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.289921224117279 + }, + "1483645336": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9307887554168701, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.13752192684190806, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.06921124458312988, + "selected_utility_gain_over_base_at_8": -1.9307887554168701, + "selector_regret_at_8": 1.9307887554168701, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9307887554168701 + }, + "1522272686": { + "base_success": 0.0, + "base_utility": 0.19844311475753784, + "branch_car_at_8": 0.03616611659526825, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28555947838938045, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.23248937726020813, + "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.19632326066493988, + "selected_utility_gain_over_base_at_8": -0.0021198540925979614, + "selector_regret_at_8": 0.03616611659526825, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7675106227397919, + "total_car_to_hidden_at_8": 1.8036767393350601 + }, + "1528740417": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2757671830981365, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1549324805": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, + "branch_car_at_8": 0.0015795694974561532, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.28625156462411294, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6785067518552145, + "proposal_oracle_utility_gain_over_base_at_8": -0.7002406964699427, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6769271823577583, + "selected_utility_gain_over_base_at_8": -0.7018202659673989, + "selector_regret_at_8": 0.0015795694974561532, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7242437700430552, + "total_car_to_hidden_at_8": 0.7258233395405114 + }, + "1607773860": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.6036040186882019, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.18975917078230178, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8267073631286621, + "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.2231033444404602, + "selected_utility_gain_over_base_at_8": -1.7768966555595398, + "selector_regret_at_8": 0.6036040186882019, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.173292636871338, + "total_car_to_hidden_at_8": 1.7768966555595398 + }, + "1665739511": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1747178030": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, + "branch_car_at_8": 0.027157030999660492, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.3089551740932353, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11134227365255356, + "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08418524265289307, + "selected_utility_gain_over_base_at_8": -0.06212134659290314, + "selector_regret_at_8": 0.027157030999660492, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8886577263474464, + "total_car_to_hidden_at_8": 1.915814757347107 + }, + "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.009028241038322449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.13674585276395504, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.034886181354522705, + "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.025857940316200256, + "selected_utility_gain_over_base_at_8": -0.006169050931930542, + "selector_regret_at_8": 0.009028241038322449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9651138186454773, + "total_car_to_hidden_at_8": 1.9741420596837997 + }, + "1919198188": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.13114443223224567, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.046244338154792786, + "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046244338154792786, + "selected_utility_gain_over_base_at_8": 0.009045466780662537, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9537556618452072, + "total_car_to_hidden_at_8": 1.9537556618452072 + }, + "1991414747": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9863876849412918, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.1306768270010748, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013612315058708191, + "selected_utility_gain_over_base_at_8": -1.9863876849412918, + "selector_regret_at_8": 1.9863876849412918, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9863876849412918 + }, + "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2554442550173647, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2015956957": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, + "branch_car_at_8": 0.022195855714380743, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.16317219295797403, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05629209652543068, + "proposal_oracle_utility_gain_over_base_at_8": -0.010827693343162536, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.034096240811049935, + "selected_utility_gain_over_base_at_8": -0.033023549057543275, + "selector_regret_at_8": 0.022195855714380743, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.2, + "success_total_car_to_hidden_at_8": 0.2, + "support_gap_at_8": 0.4735892705619335, + "total_car_to_hidden_at_8": 0.49578512627631427 + }, + "2054899126": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "211245842": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, + "branch_car_at_8": 0.6792904287576675, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07548939908426067, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8358442187309265, + "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15655378997325897, + "selected_utility_gain_over_base_at_8": -0.6822806149721146, + "selector_regret_at_8": 0.6792904287576675, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1641557812690735, + "total_car_to_hidden_at_8": 1.843446210026741 + }, + "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 0.019766509532928467, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07786951284417709, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09589323401451111, + "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07612672448158264, + "selected_utility_gain_over_base_at_8": 0.005319476127624512, + "selector_regret_at_8": 0.019766509532928467, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.904106765985489, + "total_car_to_hidden_at_8": 1.9238732755184174 + }, + "256770874": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, + "branch_car_at_8": 0.7044661641120911, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.03135306008748828, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8125, + "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10803383588790894, + "selected_utility_gain_over_base_at_8": -0.7477142214775085, + "selector_regret_at_8": 0.7044661641120911, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1875, + "total_car_to_hidden_at_8": 1.891966164112091 + }, + "278008524": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, + "branch_car_at_8": 0.011409573198761791, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.24605586147071373, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03147180867381394, + "proposal_oracle_utility_gain_over_base_at_8": -0.07600005972199142, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.020062235475052148, + "selected_utility_gain_over_base_at_8": -0.08740963292075321, + "selector_regret_at_8": 0.011409573198761791, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.082339386222884, + "total_car_to_hidden_at_8": 1.0937489594216458 + }, + "416183106": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, + "branch_car_at_8": 1.7668711841106415, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07978206783428046, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23312881588935852, + "selected_utility_gain_over_base_at_8": 0.03280240297317505, + "selector_regret_at_8": 1.7668711841106415, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7668711841106415 + }, + "534717141": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9558932781219482, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.14369576234023818, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.04410672187805176, + "selected_utility_gain_over_base_at_8": -1.9558932781219482, + "selector_regret_at_8": 1.9558932781219482, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9558932781219482 + }, + "551504635": { + "base_success": 0.0, + "base_utility": 0.7247228622436523, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2299325778712667, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7224790453910828, + "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7224790453910828, + "selected_utility_gain_over_base_at_8": -0.00224381685256958, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2775209546089172, + "total_car_to_hidden_at_8": 1.2775209546089172 + }, + "565369392": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, + "branch_car_at_8": 0.046260833740234375, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.13991651305345584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.17366480827331543, + "proposal_oracle_utility_gain_over_base_at_8": -0.6261008381843567, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.12740397453308105, + "selected_utility_gain_over_base_at_8": -0.6723616719245911, + "selector_regret_at_8": 0.046260833740234375, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2529667715231578, + "total_car_to_hidden_at_8": 1.2992276052633922 + }, + "565621826": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "585904130": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, + "branch_car_at_8": 0.012068324722349644, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07276320034061332, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032918572425842285, + "proposal_oracle_utility_gain_over_base_at_8": -0.04237352559963862, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02085024770349264, + "selected_utility_gain_over_base_at_8": -0.05444185032198826, + "selector_regret_at_8": 0.012068324722349644, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.20472841958204904, + "total_car_to_hidden_at_8": 0.2167967443043987 + }, + "626486813": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, + "branch_car_at_8": 0.02500777377281338, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.2420589895144555, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07365635968744755, + "proposal_oracle_utility_gain_over_base_at_8": -0.43178365693893284, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.04864858591463417, + "selected_utility_gain_over_base_at_8": -0.4567914307117462, + "selector_regret_at_8": 0.02500777377281338, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5414478182792664, + "total_car_to_hidden_at_8": 0.5638198243686929 + }, + "636680137": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07628525991805817, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.013928472995758057, + "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013928472995758057, + "selected_utility_gain_over_base_at_8": -1.986071527004242, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.986071527004242, + "total_car_to_hidden_at_8": 1.986071527004242 + }, + "706479600": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.16443847051324711, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "750448296": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, + "branch_car_at_8": 0.02701035887002945, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2344413298276417, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07342664152383804, + "proposal_oracle_utility_gain_over_base_at_8": -0.03810691833496094, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046416282653808594, + "selected_utility_gain_over_base_at_8": -0.06511727720499039, + "selector_regret_at_8": 0.02701035887002945, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.16238094866275787, + "total_car_to_hidden_at_8": 0.18939130753278732 + }, + "75070641": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, + "branch_car_at_8": 0.5863077491521835, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.18559462278331035, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8128874897956848, + "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22657974064350128, + "selected_utility_gain_over_base_at_8": 0.02382810413837433, + "selector_regret_at_8": 0.5863077491521835, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1871125102043152, + "total_car_to_hidden_at_8": 1.7734202593564987 + }, + "767747424": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, + "branch_car_at_8": 0.04561196081340313, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.07531396139335009, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11342880013398826, + "proposal_oracle_utility_gain_over_base_at_8": -0.4340953875798732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.06781683932058513, + "selected_utility_gain_over_base_at_8": -0.47970734839327633, + "selector_regret_at_8": 0.04561196081340313, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.08526064571924508, + "total_car_to_hidden_at_8": 0.12081881728954613 + }, + "77147264": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.2630803856693186, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.426636129617691, + "proposal_oracle_utility_gain_over_base_at_8": 0.005056649446487427, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.426636129617691, + "selected_utility_gain_over_base_at_8": 0.005056649446487427, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.573363870382309, + "total_car_to_hidden_at_8": 0.573363870382309 + }, + "794181626": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28339362113630795, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "957101489": { + "base_success": 0.0, + "base_utility": 0.20372186601161957, + "branch_car_at_8": 1.7757539749145508, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07643021229779011, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22424602508544922, + "selected_utility_gain_over_base_at_8": 0.02052415907382965, + "selector_regret_at_8": 1.7757539749145508, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7757539749145508 + }, + "998761162": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, + "branch_car_at_8": 1.9512543678283691, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28991947553571784, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04874563217163086, + "selected_utility_gain_over_base_at_8": -0.019070938229560852, + "selector_regret_at_8": 1.9512543678283691, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9512543678283691 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..bd263922de8de186c09351ff41888e14fea90c6e --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.0670691976944606, + "branch_car_at_8": 0.3250635440150897, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.21174314869082475, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.16666666666666666, + "proposal_oracle_utility_at_8": 1.3470948338508606, + "proposal_oracle_utility_gain_over_base_at_8": 0.2800256361564, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.0220312898357709, + "selected_utility_gain_over_base_at_8": -0.04503790785868963, + "selector_regret_at_8": 0.3250635440150897, + "success_selector_gap_at_8": 0.16666666666666666, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.6529051661491394, + "total_car_to_hidden_at_8": 0.977968710164229 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.45726464316248894, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, + "selector_regret_at_8": 0.45726464316248894, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.500250667333603, + "total_car_to_hidden_at_8": 1.9575153104960918 + }, + "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, + "branch_car_at_8": 0.08196123232654132, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, + "outcome_ptr_at_8": 0.29411764705882354, + "pairwise_causal_calibration_ece": 0.14892514417161398, + "proposal_oracle_success_at_8": 0.11764705882352941, + "proposal_oracle_success_gain_over_base_at_8": -0.17647058823529413, + "proposal_oracle_utility_at_8": 0.3012250267818351, + "proposal_oracle_utility_gain_over_base_at_8": -0.35000489400151896, + "selected_success_at_8": 0.08823529411764706, + "selected_success_gain_over_base_at_8": -0.20588235294117646, + "selected_utility_at_8": 0.2192637944552938, + "selected_utility_gain_over_base_at_8": -0.4319661263280603, + "selector_regret_at_8": 0.08196123232654132, + "success_selector_gap_at_8": 0.029411764705882353, + "success_support_gap_at_8": 0.20588235294117646, + "success_total_car_to_hidden_at_8": 0.23529411764705882, + "support_gap_at_8": 0.4828804947901517, + "total_car_to_hidden_at_8": 0.5630580423440894 + }, + "PushCube-v1": { + "base_success": 0.38461538461538464, + "base_utility": 0.8223185676794785, + "branch_car_at_8": 0.6069896691120588, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6153846153846154, + "pairwise_causal_calibration_ece": 0.19653272589828236, + "proposal_oracle_success_at_8": 0.7692307692307693, + "proposal_oracle_success_gain_over_base_at_8": 0.38461538461538464, + "proposal_oracle_utility_at_8": 1.55294325030767, + "proposal_oracle_utility_gain_over_base_at_8": 0.7306246826281915, + "selected_success_at_8": 0.46153846153846156, + "selected_success_gain_over_base_at_8": 0.07692307692307693, + "selected_utility_at_8": 0.9459535811956112, + "selected_utility_gain_over_base_at_8": 0.12363501351613265, + "selector_regret_at_8": 0.6069896691120588, + "success_selector_gap_at_8": 0.3076923076923077, + "success_support_gap_at_8": 0.23076923076923078, + "success_total_car_to_hidden_at_8": 0.5384615384615384, + "support_gap_at_8": 0.44705674969233, + "total_car_to_hidden_at_8": 1.0540464188043888 + }, + "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8036667982737223, + "branch_car_at_8": 0.6179056455691655, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5333333333333333, + "pairwise_causal_calibration_ece": 0.12910083675639442, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.13333333333333333, + "proposal_oracle_utility_at_8": 1.0861866732438406, + "proposal_oracle_utility_gain_over_base_at_8": 0.2825198749701182, + "selected_success_at_8": 0.13333333333333333, + "selected_success_gain_over_base_at_8": -0.06666666666666667, + "selected_utility_at_8": 0.468281027674675, + "selected_utility_gain_over_base_at_8": -0.33538577059904734, + "selector_regret_at_8": 0.6179056455691655, + "success_selector_gap_at_8": 0.2, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.8666666666666667, + "support_gap_at_8": 0.9138133267561594, + "total_car_to_hidden_at_8": 1.531718972325325 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/source_data_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed1/source_data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6cc02ca1808e0515839939b88beab0acaad5cbc --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/source_data_hash.txt @@ -0,0 +1 @@ +90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/source_split_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed1/source_split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..26988e766c6751325c7f72d33b199bd133350cde --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/source_split_hash.txt @@ -0,0 +1 @@ +867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/split_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed1/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..8533e80f235606cef4ba0119a3ac3806d63a0983 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/split_hash.txt @@ -0,0 +1 @@ +9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/table.tex b/workspace/runs/ctt_residual_rollout_val69_seed1/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..a288b0c7e4867a5949452f6ad9557dfb8b82cb0f --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/table.tex @@ -0,0 +1,28 @@ +% Auto-generated by scripts/eval_metrics.py +\begin{tabular}{lrrrr} +\toprule +Metric & N & Micro mean & CI low & CI high \\ +\midrule +base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\ +base\_utility & 69 & 0.7455 & 0.5693 & 0.9621 \\ +branch\_car\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\ +hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\ +outcome\_ptr\_at\_8 & 69 & 0.4348 & 0.3043 & 0.5362 \\ +pairwise\_causal\_calibration\_ece & 69 & 0.1583 & 0.1336 & 0.1862 \\ +proposal\_oracle\_success\_at\_8 & 69 & 0.3188 & 0.2174 & 0.4348 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.0290 & -0.0870 & 0.1449 \\ +proposal\_oracle\_utility\_at\_8 & 69 & 0.8015 & 0.6154 & 1.0082 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.0560 & -0.1555 & 0.2702 \\ +selected\_success\_at\_8 & 69 & 0.1884 & 0.1159 & 0.2899 \\ +selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.1014 & -0.2174 & 0.0000 \\ +selected\_utility\_at\_8 & 69 & 0.4776 & 0.3230 & 0.6844 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.2680 & -0.4864 & -0.0695 \\ +selector\_regret\_at\_8 & 69 & 0.3240 & 0.1877 & 0.4805 \\ +success\_selector\_gap\_at\_8 & 69 & 0.1304 & 0.0580 & 0.2174 \\ +success\_support\_gap\_at\_8 & 69 & 0.3478 & 0.2319 & 0.4638 \\ +success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4783 & 0.3623 & 0.5942 \\ +support\_gap\_at\_8 & 69 & 0.5993 & 0.4135 & 0.7763 \\ +total\_car\_to\_hidden\_at\_8 & 69 & 0.9224 & 0.7123 & 1.1263 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed1/train.log b/workspace/runs/ctt_residual_rollout_val69_seed1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0f4133e93de1098acc8dfd512625c2b03b91207d --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed1/train.log @@ -0,0 +1 @@ +CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed1/model.pt diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/data_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed2/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..a201880dd0bef2025b061d137204bb9db2f137c2 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/data_hash.txt @@ -0,0 +1 @@ +fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/eval.log b/workspace/runs/ctt_residual_rollout_val69_seed2/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..c482b6f19cac665a0e35f34db546119d302b40ef --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/eval.log @@ -0,0 +1,668 @@ +start +importing gymnasium/mani_skill +imported gymnasium/mani_skill +loaded checkpoint=runs/ctt_residual_full_seed2/model.pt +loaded charts source=301 target=69 +generated proposals rows=69 total=552 +env init sequential task=LiftPegUpright-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-484ea653bbca2de3a4a2 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-4b4cd12d70c0615e0930 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-6e268d1e2915740816a1 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-8864796e31bbe93d7c5d candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=LiftPegUpright-v1 +env init sequential task=LiftPegUpright-v1 batch=2 sim=physx_cpu render=cpu +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-b04b489f4fbad93b65ca candidate=8 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=LiftPegUpright-v1 chart=ms-c663f7e73e7e8376a05c candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=LiftPegUpright-v1 +env init sequential task=PickCube-v1 batch=1 sim=physx_cpu render=cpu +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PickCube-v1 chart=ms-2458fcfffb3f96c6ca1a candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PickCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0345d8ac530f88cefd61 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0819eb89e627595e7e1b candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-0dee962a5901fb8b5bbb candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-126eb8b36769f9ba4671 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-16661b2fd2c85f57a66c candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-20e1efbb745fd02abc95 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-2dc5138251afd26a040d candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-387e6771a55be98f1a59 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-38f7298184a2c5005e94 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-482c5c8d208b5907a2c2 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-4da03300d92420e80ed0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-57be9ae181c077e7821f candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-6388b5d2356c556c3e55 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-638d2ba1917eff4d09e0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-652ad552ba98d2736620 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-674fad0e7c042d23629e candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-728b865677134d3788e3 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7ea98b8f4d2220347ee0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-7f57045c114ddad153b5 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-81f4e8858b2ba90aea2e candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-8a72b73ac452bc190aeb candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-a96bca1ef0b74f686018 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ac4fb33f3ab4db853995 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c31b98b3d3702c6f9c8c candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c832439976a37f52d094 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-c96b415f083f60857334 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-d73fd5fce55fb58ef8e9 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-da142025733b53fcf7eb candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e2bad6e99e8a281207cc candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e30fd80adc089e9f697a candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-e9ed5d488afdfd6449d1 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ec013396e4aff6170ee1 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PullCube-v1 batch=2 sim=physx_cpu render=cpu +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-ee30ebaf98622759d888 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PullCube-v1 chart=ms-f3f1d07cfd59b823da47 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PullCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-081952a63087fc8f83cd candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-121bc2e6676c1ac10911 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-22f153f0d4fa656018e6 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-27c9f70e97b8850cc3ab candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-2867deaca7e98582620b candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-365d8c6de6de16f66744 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-3c326d05fec66e1d6bea candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-4dbdc6ed61dccebbfe92 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-58950fed04d203b97ab9 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-651bc0395d78d2819158 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-9fad82fd288e4296e354 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-b8bfdd3e6c31bd98082d candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=PushCube-v1 batch=1 sim=physx_cpu render=cpu +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=PushCube-v1 chart=ms-fc8084d5cd7da937fc02 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=PushCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3afd8959363e917591d4 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-3ff1aedb72f65a182b58 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-4048f8f6e047422e3c79 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-5b26c33feed6992386b1 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-63ac245579b1188935ad candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-6c8d06bace98a28a8e4e candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-72777024349c2aacd5e0 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-7fc5696cb3dc7b9bb9f7 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=4 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9a14552b7db695dee92c candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-9f5d57fe2153fd267774 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-b0fee26e1db967f90b81 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-bc2d3b425d9163a62dbb candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +env init sequential task=StackCube-v1 batch=3 sim=physx_cpu render=cpu +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-d1960368ef76bbb10669 candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-efdc07c985935d871adf candidate=8 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=0 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=1 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=2 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=3 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=4 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=5 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=6 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=7 shape=(1, 1, 16, 7) +execute sequential task=StackCube-v1 chart=ms-fd2387aeed9bfc9a0d08 candidate=8 shape=(1, 1, 16, 7) +batch done sequential task=StackCube-v1 +rollout complete rows=69 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics.json b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics.json index e2bb947d7607fdf401533eb46dbd5c216d22ff42..1d96232afdfcabcf2fa5b41d78de1fedadf5d1a4 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics.json @@ -5,766 +5,2050 @@ "num_rows": 69, "rows": [ { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.5006794333457947, "chart_id": "ms-484ea653bbca2de3a4a2", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10207358838016511, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7278411984443665, + "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227, "seed": "1231780997", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22716176509857178, + "selected_utility_gain_over_base_at_8": -0.49984437227249146, "selector_regret_at_8": 0.5006794333457947, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2721588015556335, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7728382349014282 }, { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 1.772506669163704, "chart_id": "ms-4b4cd12d70c0615e0930", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17388092747958847, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926, "seed": "1025740212", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22749333083629608, + "selected_utility_gain_over_base_at_8": -0.3008803278207779, "selector_regret_at_8": 1.772506669163704, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.772506669163704 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.28389310836792, "chart_id": "ms-6e268d1e2915740816a1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22214884767353088, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1475440973", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7161068916320801, + "selected_utility_gain_over_base_at_8": -1.28389310836792, "selector_regret_at_8": 1.28389310836792, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.28389310836792 }, { + "base_success": 0.0, + "base_utility": 0.7238867878913879, "branch_car_at_8": 1.273692786693573, "chart_id": "ms-8864796e31bbe93d7c5d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2702494805053396, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612, "seed": "551504635", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.726307213306427, + "selected_utility_gain_over_base_at_8": 0.0024204254150390625, "selector_regret_at_8": 1.273692786693573, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.273692786693573 }, { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.06608206033706665, "chart_id": "ms-b04b489f4fbad93b65ca", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.032532089835823584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7025249004364014, + "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276, "seed": "1113268940", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6364428400993347, + "selected_utility_gain_over_base_at_8": 0.21413031220436096, "selector_regret_at_8": 0.06608206033706665, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2974750995635986, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.3635571599006653 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.280472755432129, "chart_id": "ms-c663f7e73e7e8376a05c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16399362968800954, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1528740417", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7195272445678711, + "selected_utility_gain_over_base_at_8": -1.280472755432129, "selector_regret_at_8": 1.280472755432129, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.280472755432129 }, { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.35906313359737396, "chart_id": "ms-2458fcfffb3f96c6ca1a", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, "seed": "1273247072", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, "selector_regret_at_8": 0.35906313359737396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.5362297892570496, - "task_id": "PickCube-v1" + "task_id": "PickCube-v1", + "total_car_to_hidden_at_8": 1.8952929228544235 }, { + "base_success": 0.0, + "base_utility": 0.050487834960222244, "branch_car_at_8": 0.007528863847255707, "chart_id": "ms-0345d8ac530f88cefd61", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.062358601136824246, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09839755296707153, + "proposal_oracle_utility_gain_over_base_at_8": 0.04790971800684929, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09086868911981583, + "selected_utility_gain_over_base_at_8": 0.04038085415959358, "selector_regret_at_8": 0.007528863847255707, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.08141228556632996, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.08894114941358566 }, { + "base_success": 0.0, + "base_utility": 0.07743936777114868, "branch_car_at_8": 0.0, "chart_id": "ms-0819eb89e627595e7e1b", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.14572584629058838, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.06100934081266586, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.030317068099975586, + "proposal_oracle_utility_gain_over_base_at_8": -0.047122299671173096, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.030317068099975586, + "selected_utility_gain_over_base_at_8": -0.047122299671173096, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.11540877819061279, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.11540877819061279 }, { + "base_success": 0.0, + "base_utility": 0.19400137662887573, "branch_car_at_8": 0.02622777223587036, "chart_id": "ms-0dee962a5901fb8b5bbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15090002357562393, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11867360025644302, + "proposal_oracle_utility_gain_over_base_at_8": -0.07532777637243271, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09244582802057266, + "selected_utility_gain_over_base_at_8": -0.10155554860830307, "selector_regret_at_8": 0.02622777223587036, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.881326399743557, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9075541719794273 }, { + "base_success": 0.0, + "base_utility": 0.04583283141255379, "branch_car_at_8": 0.07351336628198624, "chart_id": "ms-126eb8b36769f9ba4671", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1985912322998047, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1968725342775377, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.15246088802814484, + "proposal_oracle_utility_gain_over_base_at_8": 0.10662805661559105, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0789475217461586, + "selected_utility_gain_over_base_at_8": 0.03311469033360481, "selector_regret_at_8": 0.07351336628198624, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.04613034427165985, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.11964371055364609 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-16661b2fd2c85f57a66c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002880295505747199, "chart_id": "ms-20e1efbb745fd02abc95", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.008822272371479922, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0008872548933140934, + "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686, "seed": "1375492786", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0005992253427393734, + "selected_utility_gain_over_base_at_8": -1.9994007746572606, "selector_regret_at_8": 0.0002880295505747199, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24256371456431225, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24285174411488697 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-2dc5138251afd26a040d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565621826", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08252689987421036, "branch_car_at_8": 0.007421374320983887, "chart_id": "ms-387e6771a55be98f1a59", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20744429528713226, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11281705834695263, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.20954282581806183, + "proposal_oracle_utility_gain_over_base_at_8": 0.12701592594385147, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.20212145149707794, + "selected_utility_gain_over_base_at_8": 0.11959455162286758, "selector_regret_at_8": 0.007421374320983887, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.005322843790054321 }, { + "base_success": 0.0, + "base_utility": 0.03917040303349495, "branch_car_at_8": 0.002210298553109169, "chart_id": "ms-38f7298184a2c5005e94", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1767096221446991, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17294274235303314, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.020024359226226807, + "proposal_oracle_utility_gain_over_base_at_8": -0.019146043807268143, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.017814060673117638, + "selected_utility_gain_over_base_at_8": -0.02135634236037731, "selector_regret_at_8": 0.002210298553109169, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1566852629184723, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15889556147158146 }, { + "base_success": 0.0, + "base_utility": 0.009124159812927246, "branch_car_at_8": 0.0006982283666729927, "chart_id": "ms-482c5c8d208b5907a2c2", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1834145486354828, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.026132711008509262, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.014287729747593403, + "proposal_oracle_utility_gain_over_base_at_8": 0.005163569934666157, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01358950138092041, + "selected_utility_gain_over_base_at_8": 0.004465341567993164, "selector_regret_at_8": 0.0006982283666729927, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16912681888788939, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.16982504725456238 }, { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.034092940390110016, "chart_id": "ms-4da03300d92420e80ed0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08595818214364334, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12480536848306656, + "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645, "seed": "1747178030", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09071242809295654, + "selected_utility_gain_over_base_at_8": -0.05559416115283966, "selector_regret_at_8": 0.034092940390110016, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8751946315169334, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9092875719070435 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.006413221824914217, "chart_id": "ms-57be9ae181c077e7821f", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.4054335986359131, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012105385772883892, + "proposal_oracle_utility_gain_over_base_at_8": -1.987894614227116, "seed": "1549324805", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.005692163947969675, + "selected_utility_gain_over_base_at_8": -1.9943078360520303, "selector_regret_at_8": 0.006413221824914217, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.987894614227116, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9943078360520303 }, { + "base_success": 0.0, + "base_utility": 0.1544450968503952, "branch_car_at_8": 0.0, "chart_id": "ms-6388b5d2356c556c3e55", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1138946652254407, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8455549031496048, "seed": "2015956957", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8455549031496048, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.06173701956868172, "branch_car_at_8": 0.016335289925336838, "chart_id": "ms-638d2ba1917eff4d09e0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.11176212877035141, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1888708945281958, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07309168577194214, + "proposal_oracle_utility_gain_over_base_at_8": 0.011354666203260422, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0567563958466053, + "selected_utility_gain_over_base_at_8": -0.004980623722076416, "selector_regret_at_8": 0.016335289925336838, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.03867044299840927, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.05500573292374611 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.001363794319331646, "chart_id": "ms-652ad552ba98d2736620", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2367449700832367, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07223239988980909, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.014465670101344585, + "proposal_oracle_utility_gain_over_base_at_8": -1.9855343298986554, "seed": "767747424", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.01310187578201294, + "selected_utility_gain_over_base_at_8": -1.986898124217987, "selector_regret_at_8": 0.001363794319331646, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2222792999818921, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22364309430122375 }, { + "base_success": 0.0, + "base_utility": 0.03784912824630737, "branch_car_at_8": 0.012949071824550629, "chart_id": "ms-674fad0e7c042d23629e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.15573063492774963, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13958284884511013, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08471894264221191, + "proposal_oracle_utility_gain_over_base_at_8": 0.04686981439590454, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07176987081766129, + "selected_utility_gain_over_base_at_8": 0.03392074257135391, "selector_regret_at_8": 0.012949071824550629, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.07101169228553772, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.08396076411008835 }, { + "base_success": 0.0, + "base_utility": 0.12791956961154938, "branch_car_at_8": 0.010773517191410065, "chart_id": "ms-728b865677134d3788e3", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.22658847272396088, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12813704483770852, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0885184183716774, + "proposal_oracle_utility_gain_over_base_at_8": -0.03940115123987198, "seed": "750448296", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07774490118026733, + "selected_utility_gain_over_base_at_8": -0.05017466843128204, "selector_regret_at_8": 0.010773517191410065, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13807005435228348, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.14884357154369354 }, { + "base_success": 0.0, + "base_utility": 0.20529556274414062, "branch_car_at_8": 0.05660251900553703, "chart_id": "ms-7ea98b8f4d2220347ee0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.27989473938941956, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12566184669419236, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11846604198217392, + "proposal_oracle_utility_gain_over_base_at_8": -0.0868295207619667, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06186352297663689, + "selected_utility_gain_over_base_at_8": -0.14343203976750374, "selector_regret_at_8": 0.05660251900553703, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16142869740724564, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.21803121641278267 }, { + "base_success": 0.0, + "base_utility": 0.18213583528995514, "branch_car_at_8": 0.003586272243410349, "chart_id": "ms-7f57045c114ddad153b5", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.227622389793396, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22935436227797687, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.00681296968832612, + "proposal_oracle_utility_gain_over_base_at_8": -0.17532286560162902, "seed": "278008524", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0032266974449157715, + "selected_utility_gain_over_base_at_8": -0.17890913784503937, "selector_regret_at_8": 0.003586272243410349, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.22080942010506988, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22439569234848022 }, { + "base_success": 0.0, + "base_utility": 0.0056487321853637695, "branch_car_at_8": 0.001928985584527254, "chart_id": "ms-81f4e8858b2ba90aea2e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1631626933813095, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1734428620262507, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.008817712776362896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0031689805909991264, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006888727191835642, + "selected_utility_gain_over_base_at_8": 0.0012399950064718723, "selector_regret_at_8": 0.001928985584527254, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1543449806049466, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15627396618947387 }, { + "base_success": 0.0, + "base_utility": 0.00698717450723052, "branch_car_at_8": 0.08708053827285767, "chart_id": "ms-8a72b73ac452bc190aeb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.10329639911651611, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.5629214026808478, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.10641509294509888, + "proposal_oracle_utility_gain_over_base_at_8": 0.09942791843786836, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01933455467224121, + "selected_utility_gain_over_base_at_8": 0.01234738016501069, "selector_regret_at_8": 0.08708053827285767, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0839618444442749 }, { + "base_success": 0.0, + "base_utility": 0.08285123109817505, "branch_car_at_8": 0.032952308654785156, "chart_id": "ms-a96bca1ef0b74f686018", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1572066992521286, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.34632898516451505, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06243818998336792, + "proposal_oracle_utility_gain_over_base_at_8": -0.02041304111480713, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.029485881328582764, + "selected_utility_gain_over_base_at_8": -0.053365349769592285, "selector_regret_at_8": 0.032952308654785156, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09476850926876068, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.12772081792354584 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.05388810485601425, "chart_id": "ms-ac4fb33f3ab4db853995", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.24514405918899088, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.16087166965007782, + "proposal_oracle_utility_gain_over_base_at_8": -1.8391283303499222, "seed": "565369392", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.10698356479406357, + "selected_utility_gain_over_base_at_8": -1.8930164352059364, "selector_regret_at_8": 0.05388810485601425, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8391283303499222, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8930164352059364 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c31b98b3d3702c6f9c8c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "2054899126", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.037537217140197754, "branch_car_at_8": 0.03309768531471491, "chart_id": "ms-c832439976a37f52d094", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2594742774963379, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.34007124070568584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04554198309779167, + "proposal_oracle_utility_gain_over_base_at_8": 0.008004765957593918, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012444297783076763, + "selected_utility_gain_over_base_at_8": -0.02509291935712099, "selector_regret_at_8": 0.03309768531471491, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.21393229439854622, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24702997971326113 }, { + "base_success": 0.0, + "base_utility": 0.09138330072164536, "branch_car_at_8": 0.0, "chart_id": "ms-c96b415f083f60857334", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.26426148414611816, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2676934704052151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0532812662422657, + "proposal_oracle_utility_gain_over_base_at_8": -0.038102034479379654, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0532812662422657, + "selected_utility_gain_over_base_at_8": -0.038102034479379654, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.21098021790385246, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.21098021790385246 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-d73fd5fce55fb58ef8e9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2870871533378466, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "534717141", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.14809735119342804, "branch_car_at_8": 0.0010104775428771973, "chart_id": "ms-da142025733b53fcf7eb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2643269896507263, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17434085305175193, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.001866757869720459, + "proposal_oracle_utility_gain_over_base_at_8": -0.14623059332370758, "seed": "141984615", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0008562803268432617, + "selected_utility_gain_over_base_at_8": -0.14724107086658478, "selector_regret_at_8": 0.0010104775428771973, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.26246023178100586, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.26347070932388306 }, { + "base_success": 0.0, + "base_utility": 0.1362423449754715, "branch_car_at_8": 0.03072514245286584, "chart_id": "ms-e2bad6e99e8a281207cc", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20825156569480896, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3105160115580491, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03793333098292351, + "proposal_oracle_utility_gain_over_base_at_8": -0.09830901399254799, "seed": "1549324805", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007208188530057669, + "selected_utility_gain_over_base_at_8": -0.12903415644541383, "selector_regret_at_8": 0.03072514245286584, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.17031823471188545, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2010433771647513 }, { + "base_success": 0.0, + "base_utility": 0.09695577621459961, "branch_car_at_8": 0.0, "chart_id": "ms-e30fd80adc089e9f697a", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18920521438121796, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17993304523315273, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13661859929561615, + "proposal_oracle_utility_gain_over_base_at_8": 0.03966282308101654, "seed": "585904130", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13661859929561615, + "selected_utility_gain_over_base_at_8": 0.03966282308101654, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.05258661508560181, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.05258661508560181 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.03718312829732895, "chart_id": "ms-e9ed5d488afdfd6449d1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17997058384239856, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.1506197303533554, + "proposal_oracle_utility_gain_over_base_at_8": -1.8493802696466446, "seed": "626486813", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.11343660205602646, + "selected_utility_gain_over_base_at_8": -1.8865633979439735, "selector_regret_at_8": 0.03718312829732895, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8493802696466446, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8865633979439735 }, { + "base_success": 0.0, + "base_utility": 0.09514755010604858, "branch_car_at_8": 0.024643758311867714, "chart_id": "ms-ec013396e4aff6170ee1", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24502670764923096, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15485012388319694, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04536725953221321, + "proposal_oracle_utility_gain_over_base_at_8": -0.04978029057383537, "seed": "750448296", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.020723501220345497, + "selected_utility_gain_over_base_at_8": -0.07442404888570309, "selector_regret_at_8": 0.024643758311867714, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.19965944811701775, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22430320642888546 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.006491700187325478, "chart_id": "ms-ee30ebaf98622759d888", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12019361687075947, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.02436021901667118, + "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288, "seed": "636680137", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.017868518829345703, + "selected_utility_gain_over_base_at_8": -1.9821314811706543, "selector_regret_at_8": 0.006491700187325478, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9756397809833288, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9821314811706543 }, { + "base_success": 0.0, + "base_utility": 0.026694953441619873, "branch_car_at_8": 0.012709278613328934, "chart_id": "ms-f3f1d07cfd59b823da47", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.17124073207378387, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.22819684948441313, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06399381160736084, + "proposal_oracle_utility_gain_over_base_at_8": 0.03729885816574097, "seed": "2015956957", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.051284532994031906, + "selected_utility_gain_over_base_at_8": 0.024589579552412033, "selector_regret_at_8": 0.012709278613328934, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.10724692046642303, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.11995619907975197 }, { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 1.861636370420456, "chart_id": "ms-081952a63087fc8f83cd", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09903289408743085, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419, "seed": "2141802816", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13836362957954407, + "selected_utility_gain_over_base_at_8": 0.06755638122558594, "selector_regret_at_8": 1.861636370420456, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.861636370420456 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-121bc2e6676c1ac10911", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "77147264", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-22f153f0d4fa656018e6", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.28406274395520603, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1271718007", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.024854734539985657, "chart_id": "ms-27c9f70e97b8850cc3ab", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2146835718150399, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.043482705950737, + "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201, "seed": "1881223807", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018627971410751343, + "selected_utility_gain_over_base_at_8": -0.013399019837379456, "selector_regret_at_8": 0.024854734539985657, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.956517294049263, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9813720285892487 }, { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, "chart_id": "ms-2867deaca7e98582620b", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, "seed": "1189392917", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, "chart_id": "ms-365d8c6de6de16f66744", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.14617959762414967, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, "seed": "1997090716", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03280790150165558, "branch_car_at_8": 0.036816492676734924, "chart_id": "ms-3c326d05fec66e1d6bea", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.08134846381937184, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05891832709312439, + "proposal_oracle_utility_gain_over_base_at_8": 0.02611042559146881, "seed": "278008524", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.022101834416389465, + "selected_utility_gain_over_base_at_8": -0.010706067085266113, "selector_regret_at_8": 0.036816492676734924, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9410816729068756, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9778981655836105 }, { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, "chart_id": "ms-4dbdc6ed61dccebbfe92", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, "seed": "794181626", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-58950fed04d203b97ab9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.233029795067053, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1113371261", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9686058908700943, "chart_id": "ms-651bc0395d78d2819158", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.5995435922806452, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1483645336", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0313941091299057, + "selected_utility_gain_over_base_at_8": -1.9686058908700943, "selector_regret_at_8": 1.9686058908700943, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9686058908700943 }, { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, "chart_id": "ms-9fad82fd288e4296e354", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17468938592004596, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, "seed": "1176943004", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.939187914133072, "chart_id": "ms-b8bfdd3e6c31bd98082d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11706618974004251, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, "seed": "998761162", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0608120858669281, + "selected_utility_gain_over_base_at_8": -0.007004484534263611, "selector_regret_at_8": 1.939187914133072, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.939187914133072 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-fc8084d5cd7da937fc02", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11896663871153935, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1991414747", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.3710349500179291, "chart_id": "ms-3afd8959363e917591d4", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09051828052202912, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5650836229324341, + "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713, "seed": "75070641", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.194048672914505, + "selected_utility_gain_over_base_at_8": -0.008702963590621948, "selector_regret_at_8": 0.3710349500179291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.434916377067566, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.805951327085495 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9218642711639404, "chart_id": "ms-3ff1aedb72f65a182b58", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19289744165908876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1055567618", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07813572883605957, + "selected_utility_gain_over_base_at_8": -1.9218642711639404, "selector_regret_at_8": 1.9218642711639404, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9218642711639404 }, { + "base_success": 0.0, + "base_utility": 0.19648407399654388, "branch_car_at_8": 0.03480546176433563, "chart_id": "ms-4048f8f6e047422e3c79", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1805709977888097, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22467364370822906, + "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518, "seed": "1055157354", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18986818194389343, + "selected_utility_gain_over_base_at_8": -0.006615892052650452, "selector_regret_at_8": 0.03480546176433563, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.775326356291771, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8101318180561066 }, { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.017972871661186218, "chart_id": "ms-5b26c33feed6992386b1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.3276128524355476, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1949000507593155, + "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294, "seed": "1522272686", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.17692717909812927, + "selected_utility_gain_over_base_at_8": -0.01783391833305359, "selector_regret_at_8": 0.017972871661186218, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8050999492406845, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8230728209018707 }, { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.0, "chart_id": "ms-63ac245579b1188935ad", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13508468824583414, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265, "seed": "211245842", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1611655950546265, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-6c8d06bace98a28a8e4e", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2508232858898205, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1665739511", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.046216949820518494, "chart_id": "ms-72777024349c2aacd5e0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046630940401554466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.21061499416828156, + "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359, "seed": "256770874", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16439804434776306, + "selected_utility_gain_over_base_at_8": -0.6913500130176544, "selector_regret_at_8": 0.046216949820518494, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7893850058317184, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.835601955652237 }, { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 1.432861328125, "chart_id": "ms-7fc5696cb3dc7b9bb9f7", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.21219243624964876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252, "seed": "957101489", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.567138671875, + "selected_utility_gain_over_base_at_8": 0.3631737232208252, "selector_regret_at_8": 1.432861328125, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.432861328125 }, { + "base_success": 0.0, + "base_utility": 0.8431589603424072, "branch_car_at_8": 0.0, "chart_id": "ms-9a14552b7db695dee92c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.24618369758422665, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1568410396575928, "seed": "77147264", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1568410396575928, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-9f5d57fe2153fd267774", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.24177237819615016, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1607773860", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 1.90096914768219, "chart_id": "ms-b0fee26e1db967f90b81", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.32948159974174274, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187, "seed": "1358421325", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09903085231781006, + "selected_utility_gain_over_base_at_8": -0.7336714267730713, "selector_regret_at_8": 1.90096914768219, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.90096914768219 }, { + "base_success": 0.0, + "base_utility": 0.8307622671127319, "branch_car_at_8": 0.39469505846500397, "chart_id": "ms-bc2d3b425d9163a62dbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18608118833691456, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5271345376968384, + "proposal_oracle_utility_gain_over_base_at_8": -0.30362772941589355, "seed": "1271718007", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1324394792318344, + "selected_utility_gain_over_base_at_8": -0.6983227878808975, "selector_regret_at_8": 0.39469505846500397, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.4728654623031616, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8675605207681656 }, { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.812951534986496, "chart_id": "ms-d1960368ef76bbb10669", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18072825209943494, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, "seed": "416183106", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18704846501350403, + "selected_utility_gain_over_base_at_8": -0.013277947902679443, "selector_regret_at_8": 1.812951534986496, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.812951534986496 }, { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, "chart_id": "ms-efdc07c985935d871adf", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.22468613298280815, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, "seed": "706479600", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.018839523196220398, "chart_id": "ms-fd2387aeed9bfc9a0d08", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1669540555093941, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06275103986263275, + "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502, "seed": "1919198188", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043911516666412354, + "selected_utility_gain_over_base_at_8": 0.0067126452922821045, "selector_regret_at_8": 0.018839523196220398, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9372489601373672, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9560884833335876 } ], "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4521276595744681, + "low": 0.20921985815602834, + "mean": 0.3297872340425532, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3297872340425532, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.33815987933634994, + "low": 0.10666666666666666, + "mean": 0.24241327300150828, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24241327300150828, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.42028985507246375, + "low": 0.18840579710144928, + "mean": 0.2898550724637681, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0924235453427908, + "low": 0.6323911708297458, + "mean": 0.8568741302496377, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8568741302496377, + "per_seed": { + "1025740212": 0.528373658657074, + "1055157354": 0.19648407399654388, + "1055567618": 2.0, + "1113268940": 0.42231252789497375, + "1113371261": 2.0, + "1176943004": 0.05094052851200104, + "1189392917": 0.10841864347457886, + "1231780997": 0.7270061373710632, + "1271718007": 1.415381133556366, + "1273247072": 0.15342965722084045, + "1358421325": 0.8327022790908813, + "1375492786": 2.0, + "141984615": 0.09381213908394177, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.19476109743118286, + "1528740417": 2.0, + "1549324805": 1.3787474483251572, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.1463065892457962, + "1881223807": 0.0320269912481308, + "1919198188": 0.03719887137413025, + "1991414747": 2.0, + "1997090716": 0.13927479088306427, + "2015956957": 0.06711978986859321, + "2054899126": 2.0, + "211245842": 0.8388344049453735, + "2141802816": 0.07080724835395813, + "256770874": 0.8557480573654175, + "278008524": 0.10747186839580536, + "416183106": 0.20032641291618347, + "534717141": 2.0, + "551504635": 0.7238867878913879, + "565369392": 0.7997656464576721, + "565621826": 2.0, + "585904130": 0.07529209802548091, + "626486813": 0.5054400166263804, + "636680137": 2.0, + "706479600": 0.8350887894630432, + "750448296": 0.11153355985879898, + "75070641": 0.20275163650512695, + "767747424": 0.5475241877138615, + "77147264": 1.4215794801712036, + "794181626": 0.18095605075359344, + "957101489": 0.2039649486541748, + "998761162": 0.06781657040119171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9313609859940832, + "low": 0.4139719756444295, + "mean": 0.6996295724218208, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6996295724218208, + "per_task": { + "LiftPegUpright-v1": 1.066929851969083, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.8217729788560134, + "StackCube-v1": 0.8047854532798131 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9619567723796312, + "low": 0.5695875146345276, + "mean": 0.7456761826086196, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, "branch_car_at_8": { "macro_by_seed": { "ci": { @@ -855,6 +2139,186 @@ }, "n": 69 }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.922695035460993, + "low": 0.7351063829787235, + "mean": 0.8358156028368794, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8358156028368794, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.782608695652174, + "low": 0.5507246376811594, + "mean": 0.6666666666666666, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8615907955016442, + "low": 1.529367521088174, + "mean": 1.7062028111638965, + "n": 47, + "num_samples": 500 + }, + "mean": 1.7062028111638965, + "per_seed": { + "1025740212": 2.0, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 2.0, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1358421325": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.20044784247875214, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 2.0, + "1528740417": 2.0, + "1549324805": 1.4027505218982697, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 2.0, + "1881223807": 2.0, + "1919198188": 2.0, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.5298813670873642, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 2.0, + "278008524": 1.113811194896698, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 1.4266315797964733, + "565621826": 2.0, + "585904130": 0.23764699200789133, + "626486813": 0.6124684102833271, + "636680137": 2.0, + "706479600": 2.0, + "750448296": 0.23580759018659592, + "75070641": 2.0, + "767747424": 0.18863565661013126, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.6062410234109215, + "low": 1.1855228276572365, + "mean": 1.3999846732054932, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, "outcome_ptr_at_8": { "macro_by_seed": { "ci": { @@ -1035,134 +2499,1124 @@ }, "n": 69 }, - "selector_regret_at_8": { + "proposal_oracle_success_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.6595998992103133, - "low": 0.2230010245754945, - "mean": 0.43210832736758736, + "high": 0.6808510638297872, + "low": 0.425531914893617, + "mean": 0.5539007092198581, "n": 47, "num_samples": 500 }, - "mean": 0.43210832736758736, + "mean": 0.5539007092198581, "per_seed": { - "1025740212": 1.772506669163704, - "1055157354": 0.03480546176433563, - "1055567618": 1.9218642711639404, - "1113268940": 0.06608206033706665, - "1113371261": 0.0, - "1176943004": 0.0, - "1189392917": 0.0, - "1231780997": 0.5006794333457947, - "1271718007": 0.19734752923250198, - "1273247072": 0.35906313359737396, - "1358421325": 1.90096914768219, - "1375492786": 0.0002880295505747199, - "141984615": 0.013830550014972687, - "1475440973": 1.28389310836792, - "1483645336": 1.9686058908700943, - "1522272686": 0.017972871661186218, - "1528740417": 1.280472755432129, - "1549324805": 0.01237945475926002, - "1607773860": 0.0, - "1665739511": 0.0, - "1747178030": 0.034092940390110016, - "1881223807": 0.024854734539985657, - "1919198188": 0.018839523196220398, - "1991414747": 0.0, - "1997090716": 0.0, - "2015956957": 0.005573729798197746, - "2054899126": 0.0, - "211245842": 0.0, - "2141802816": 1.861636370420456, - "256770874": 0.046216949820518494, - "278008524": 0.020201382460072637, - "416183106": 1.812951534986496, - "534717141": 0.0, - "551504635": 1.273692786693573, - "565369392": 0.04557279869914055, - "565621826": 0.0, - "585904130": 0.011032561771571636, - "626486813": 0.031722720130346715, - "636680137": 0.006491700187325478, - "706479600": 0.0, - "750448296": 0.01770863775163889, - "75070641": 0.3710349500179291, - "767747424": 0.024658456211909652, - "77147264": 0.0, - "794181626": 0.0, - "957101489": 1.432861328125, - "998761162": 1.939187914133072 + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 } }, "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.7792545604018064, - "low": 0.18968407530785322, - "mean": 0.476884111007974, + "high": 0.7117948717948719, + "low": 0.14941176470588236, + "mean": 0.4519758672699849, "n": 5, "num_samples": 500 }, - "mean": 0.476884111007974, + "mean": 0.4519758672699849, "per_task": { - "LiftPegUpright-v1": 1.029554468890031, - "PickCube-v1": 0.35906313359737396, - "PullCube-v1": 0.017109284352045506, - "PushCube-v1": 0.4485462617415648, - "StackCube-v1": 0.5301474064588547 + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.14705882352941177, + "PushCube-v1": 0.8461538461538461, + "StackCube-v1": 0.6 } }, - "metric": "selector_regret_at_8", + "metric": "proposal_oracle_success_at_8", "micro": { "confidence": 0.95, - "high": 0.46172967990117986, - "low": 0.1641734795007126, - "mean": 0.3029191031076851, + "high": 0.5507246376811594, + "low": 0.3188405797101449, + "mean": 0.42028985507246375, "n": 69, "num_samples": 500 }, "n": 69 }, - "support_gap_at_8": { + "proposal_oracle_success_gain_over_base_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.7286775349375151, - "low": 0.30572481987938444, - "mean": 0.5087829326731087, + "high": 0.35496453900709213, + "low": 0.08120567375886525, + "mean": 0.224113475177305, "n": 47, "num_samples": 500 }, - "mean": 0.5087829326731087, + "mean": 0.224113475177305, "per_seed": { - "1025740212": 0.0, - "1055157354": 1.775326356291771, + "1025740212": 1.0, + "1055157354": 0.0, "1055567618": 0.0, - "1113268940": 1.2974750995635986, + "1113268940": 0.0, "1113371261": 0.0, - "1176943004": 0.0, - "1189392917": 0.0, - "1231780997": 1.2721588015556335, - "1271718007": 0.7364327311515808, - "1273247072": 1.5362297892570496, - "1358421325": 0.0, - "1375492786": 0.24256371456431225, - "141984615": 0.14621367553869882, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 1.0, + "1375492786": -1.0, + "141984615": 0.0, "1475440973": 0.0, "1483645336": 0.0, - "1522272686": 1.8050999492406845, + "1522272686": 0.0, "1528740417": 0.0, - "1549324805": 0.7194042829796672, + "1549324805": -0.3333333333333333, "1607773860": 0.0, "1665739511": 0.0, - "1747178030": 1.8751946315169334, - "1881223807": 1.956517294049263, - "1919198188": 1.9372489601373672, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, "1991414747": 0.0, - "1997090716": 0.0, - "2015956957": 0.09007053077220917, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4235897435897436, + "low": -0.0215686274509804, + "mean": 0.20956259426847662, + "n": 5, + "num_samples": 500 + }, + "mean": 0.20956259426847662, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.14705882352941177, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.4 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2608695652173913, + "low": -0.014492753623188406, + "mean": 0.13043478260869565, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4388152989261644, + "low": 0.9575430694977415, + "mean": 1.1974476296841894, + "n": 47, + "num_samples": 500 + }, + "mean": 1.1974476296841894, + "per_seed": { + "1025740212": 2.0, + "1055157354": 0.22467364370822906, + "1055567618": 2.0, + "1113268940": 0.7025249004364014, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.7278411984443665, + "1271718007": 1.2635672688484192, + "1273247072": 0.46377021074295044, + "1358421325": 2.0, + "1375492786": 0.0008872548933140934, + "141984615": 0.054234166940053306, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.1949000507593155, + "1528740417": 2.0, + "1549324805": 0.6833462389186025, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.12480536848306656, + "1881223807": 0.043482705950737, + "1919198188": 0.06275103986263275, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.43981083631515505, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 0.21061499416828156, + "278008524": 0.032865648390725255, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 0.1326704372962316, + "565621826": 2.0, + "585904130": 0.07848061621189117, + "626486813": 0.07003506645560265, + "636680137": 0.02436021901667118, + "706479600": 2.0, + "750448296": 0.0669428389519453, + "75070641": 0.5650836229324341, + "767747424": 0.11239026742987335, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5220236249459096, + "low": 0.5982443927517891, + "mean": 1.081913336966832, + "n": 5, + "num_samples": 500 + }, + "mean": 1.081913336966832, + "per_task": { + "LiftPegUpright-v1": 1.5717276831467946, + "PickCube-v1": 0.46377021074295044, + "PullCube-v1": 0.3548735701530377, + "PushCube-v1": 1.7001846948495278, + "StackCube-v1": 1.3190105259418488 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.1545684401146676, + "low": 0.7270322287424832, + "mean": 0.9253245886521535, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5855443862486458, + "low": 0.08830777057388746, + "mean": 0.3405734994345517, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3405734994345517, + "per_seed": { + "1025740212": 1.471626341342926, + "1055157354": 0.02818956971168518, + "1055567618": 0.0, + "1113268940": 0.2802123725414276, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": 0.0008350610733032227, + "1271718007": -0.15181386470794678, + "1273247072": 0.31034055352211, + "1358421325": 1.1672977209091187, + "1375492786": -1.999112745106686, + "141984615": -0.039577972143888474, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0001389533281326294, + "1528740417": 0.0, + "1549324805": -0.6954012094065547, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": -0.021501220762729645, + "1881223807": 0.011455714702606201, + "1919198188": 0.025552168488502502, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": 0.3726910464465618, + "2054899126": 0.0, + "211245842": 1.1611655950546265, + "2141802816": 1.9291927516460419, + "256770874": -0.6451330631971359, + "278008524": -0.0746062200050801, + "416183106": 1.7996735870838165, + "534717141": 0.0, + "551504635": 1.276113212108612, + "565369392": -0.6670952091614405, + "565621826": 0.0, + "585904130": 0.0031885181864102683, + "626486813": -0.43540495017077774, + "636680137": -1.9756397809833288, + "706479600": 1.1649112105369568, + "750448296": -0.044590720906853676, + "75070641": 0.36233198642730713, + "767747424": -0.4351339202839881, + "77147264": 0.5784205198287964, + "794181626": 1.8190439492464066, + "957101489": 1.7960350513458252, + "998761162": 1.9321834295988083 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6900747065360731, + "low": -0.01290068514136069, + "mean": 0.382283764545011, + "n": 5, + "num_samples": 500 + }, + "mean": 0.382283764545011, + "per_task": { + "LiftPegUpright-v1": 0.5047978311777115, + "PickCube-v1": 0.31034055352211, + "PullCube-v1": -0.29635635063031635, + "PushCube-v1": 0.8784117159935144, + "StackCube-v1": 0.5142250726620357 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.40019721312421386, + "low": -0.055687820795329586, + "mean": 0.179648406043534, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4482269503546099, + "low": 0.19148936170212766, + "mean": 0.3198581560283688, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3198581560283688, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.43589743589743596, + "low": 0.058823529411764705, + "mean": 0.21915535444947212, + "n": 5, + "num_samples": 500 + }, + "mean": 0.21915535444947212, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.14705882352941177, + "PushCube-v1": 0.6153846153846154, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.36231884057971014, + "low": 0.17391304347826086, + "mean": 0.2608695652173913, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.11950354609929079, + "low": -0.16241134751773048, + "mean": -0.0099290780141844, + "n": 47, + "num_samples": 500 + }, + "mean": -0.0099290780141844, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": -1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": -1.0, + "1483645336": -1.0, + "1522272686": 0.0, + "1528740417": -1.0, + "1549324805": -0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.14564102564102566, + "low": -0.20274509803921567, + "mean": -0.02325791855203619, + "n": 5, + "num_samples": 500 + }, + "mean": -0.02325791855203619, + "per_task": { + "LiftPegUpright-v1": -0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.14705882352941177, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.13333333333333333 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.08695652173913043, + "low": -0.15942028985507245, + "mean": -0.028985507246376812, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9970987782217497, + "low": 0.5411203128707609, + "mean": 0.7653393023166019, + "n": 47, + "num_samples": 500 + }, + "mean": 0.7653393023166019, + "per_seed": { + "1025740212": 0.22749333083629608, + "1055157354": 0.18986818194389343, + "1055567618": 0.07813572883605957, + "1113268940": 0.6364428400993347, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.22716176509857178, + "1271718007": 1.0662197396159172, + "1273247072": 0.10470707714557648, + "1358421325": 0.09903085231781006, + "1375492786": 0.0005992253427393734, + "141984615": 0.04040361692508062, + "1475440973": 0.7161068916320801, + "1483645336": 0.0313941091299057, + "1522272686": 0.17692717909812927, + "1528740417": 0.7195272445678711, + "1549324805": 0.6709667841593424, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.09071242809295654, + "1881223807": 0.018627971410751343, + "1919198188": 0.043911516666412354, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.4342371065169573, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 0.13836362957954407, + "256770874": 0.16439804434776306, + "278008524": 0.012664265930652618, + "416183106": 0.18704846501350403, + "534717141": 2.0, + "551504635": 0.726307213306427, + "565369392": 0.08709763859709103, + "565621826": 2.0, + "585904130": 0.06744805444031954, + "626486813": 0.03831234632525593, + "636680137": 0.017868518829345703, + "706479600": 2.0, + "750448296": 0.049234201200306416, + "75070641": 0.194048672914505, + "767747424": 0.0877318112179637, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 0.567138671875, + "998761162": 0.0608120858669281 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9739732449329817, + "low": 0.2881497273441431, + "mean": 0.6050292259588579, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6050292259588579, + "per_task": { + "LiftPegUpright-v1": 0.5421732142567635, + "PickCube-v1": 0.10470707714557648, + "PullCube-v1": 0.3377642858009922, + "PushCube-v1": 1.251638433107963, + "StackCube-v1": 0.788863119482994 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.8055619765385189, + "low": 0.44728328000781115, + "mean": 0.6224054855444685, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.13649843921744537, + "low": -0.349154780383126, + "mean": -0.09153482793303577, + "n": 47, + "num_samples": 500 + }, + "mean": -0.09153482793303577, + "per_seed": { + "1025740212": -0.3008803278207779, + "1055157354": -0.006615892052650452, + "1055567618": -1.9218642711639404, + "1113268940": 0.21413031220436096, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": -0.49984437227249146, + "1271718007": -0.34916139394044876, + "1273247072": -0.04872258007526398, + "1358421325": -0.7336714267730713, + "1375492786": -1.9994007746572606, + "141984615": -0.05340852215886116, + "1475440973": -1.28389310836792, + "1483645336": -1.9686058908700943, + "1522272686": -0.01783391833305359, + "1528740417": -1.280472755432129, + "1549324805": -0.7077806641658148, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": -0.05559416115283966, + "1881223807": -0.013399019837379456, + "1919198188": 0.0067126452922821045, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": 0.36711731664836406, + "2054899126": 0.0, + "211245842": 1.1611655950546265, + "2141802816": 0.06755638122558594, + "256770874": -0.6913500130176544, + "278008524": -0.09480760246515274, + "416183106": -0.013277947902679443, + "534717141": 0.0, + "551504635": 0.0024204254150390625, + "565369392": -0.712668007860581, + "565621826": 0.0, + "585904130": -0.007844043585161367, + "626486813": -0.46712767030112445, + "636680137": -1.9821314811706543, + "706479600": 1.1649112105369568, + "750448296": -0.062299358658492565, + "75070641": -0.008702963590621948, + "767747424": -0.45979237649589777, + "77147264": 0.5784205198287964, + "794181626": 1.8190439492464066, + "957101489": 0.3631737232208252, + "998761162": -0.007004484534263611 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1623927814226884, + "low": -0.3557238355283534, + "mean": -0.09460034646296299, + "n": 5, + "num_samples": 500 + }, + "mean": -0.09460034646296299, + "per_task": { + "LiftPegUpright-v1": -0.5247566377123197, + "PickCube-v1": -0.04872258007526398, + "PullCube-v1": -0.31346563498236185, + "PushCube-v1": 0.4298654542519496, + "StackCube-v1": -0.015922333796819052 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.08361495469791302, + "low": -0.3535496463047583, + "mean": -0.1232706970641511, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6595998992103133, + "low": 0.2230010245754945, + "mean": 0.43210832736758736, + "n": 47, + "num_samples": 500 + }, + "mean": 0.43210832736758736, + "per_seed": { + "1025740212": 1.772506669163704, + "1055157354": 0.03480546176433563, + "1055567618": 1.9218642711639404, + "1113268940": 0.06608206033706665, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.5006794333457947, + "1271718007": 0.19734752923250198, + "1273247072": 0.35906313359737396, + "1358421325": 1.90096914768219, + "1375492786": 0.0002880295505747199, + "141984615": 0.013830550014972687, + "1475440973": 1.28389310836792, + "1483645336": 1.9686058908700943, + "1522272686": 0.017972871661186218, + "1528740417": 1.280472755432129, + "1549324805": 0.01237945475926002, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.034092940390110016, + "1881223807": 0.024854734539985657, + "1919198188": 0.018839523196220398, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.005573729798197746, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.861636370420456, + "256770874": 0.046216949820518494, + "278008524": 0.020201382460072637, + "416183106": 1.812951534986496, + "534717141": 0.0, + "551504635": 1.273692786693573, + "565369392": 0.04557279869914055, + "565621826": 0.0, + "585904130": 0.011032561771571636, + "626486813": 0.031722720130346715, + "636680137": 0.006491700187325478, + "706479600": 0.0, + "750448296": 0.01770863775163889, + "75070641": 0.3710349500179291, + "767747424": 0.024658456211909652, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.432861328125, + "998761162": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7792545604018064, + "low": 0.18968407530785322, + "mean": 0.476884111007974, + "n": 5, + "num_samples": 500 + }, + "mean": 0.476884111007974, + "per_task": { + "LiftPegUpright-v1": 1.029554468890031, + "PickCube-v1": 0.35906313359737396, + "PullCube-v1": 0.017109284352045506, + "PushCube-v1": 0.4485462617415648, + "StackCube-v1": 0.5301474064588547 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.46172967990117986, + "low": 0.1641734795007126, + "mean": 0.3029191031076851, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3617021276595745, + "low": 0.10638297872340426, + "mean": 0.23404255319148937, + "n": 47, + "num_samples": 500 + }, + "mean": 0.23404255319148937, + "per_seed": { + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4461538461538462, + "low": 0.046153846153846156, + "mean": 0.2328205128205128, + "n": 5, + "num_samples": 500 + }, + "mean": 0.2328205128205128, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.26666666666666666 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2608695652173913, + "low": 0.08695652173913043, + "mean": 0.15942028985507245, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4148936170212766, + "low": 0.1595744680851064, + "mean": 0.2819148936170212, + "n": 47, + "num_samples": 500 + }, + "mean": 0.2819148936170212, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7152941176470589, + "low": 0.19879336349924587, + "mean": 0.4127300150829562, + "n": 5, + "num_samples": 500 + }, + "mean": 0.4127300150829562, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.17647058823529413, + "PushCube-v1": 0.15384615384615385, + "StackCube-v1": 0.4 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.34782608695652173, + "low": 0.14492753623188406, + "mean": 0.2463768115942029, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6524822695035459, + "low": 0.3776595744680851, + "mean": 0.5159574468085107, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5159574468085107, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9333333333333332, + "low": 0.3161387631975867, + "mean": 0.645550527903469, + "n": 5, + "num_samples": 500 + }, + "mean": 0.645550527903469, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.17647058823529413, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5217391304347826, + "low": 0.2753623188405797, + "mean": 0.4057971014492754, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7286775349375151, + "low": 0.30572481987938444, + "mean": 0.5087829326731087, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5087829326731087, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.775326356291771, + "1055567618": 0.0, + "1113268940": 1.2974750995635986, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.2721588015556335, + "1271718007": 0.7364327311515808, + "1273247072": 1.5362297892570496, + "1358421325": 0.0, + "1375492786": 0.24256371456431225, + "141984615": 0.14621367553869882, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.8050999492406845, + "1528740417": 0.0, + "1549324805": 0.7194042829796672, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.8751946315169334, + "1881223807": 1.956517294049263, + "1919198188": 1.9372489601373672, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.09007053077220917, "2054899126": 0.0, "211245842": 0.0, "2141802816": 0.0, @@ -1214,6 +3668,96 @@ "num_samples": 500 }, "n": 69 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1787349130094698, + "low": 0.7077262575365768, + "mean": 0.9408635088472943, + "n": 47, + "num_samples": 500 + }, + "mean": 0.9408635088472943, + "per_seed": { + "1025740212": 1.772506669163704, + "1055157354": 1.8101318180561066, + "1055567618": 1.9218642711639404, + "1113268940": 1.3635571599006653, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.7728382349014282, + "1271718007": 0.9337802603840828, + "1273247072": 1.8952929228544235, + "1358421325": 1.90096914768219, + "1375492786": 0.24285174411488697, + "141984615": 0.1600442255536715, + "1475440973": 1.28389310836792, + "1483645336": 1.9686058908700943, + "1522272686": 1.8230728209018707, + "1528740417": 1.280472755432129, + "1549324805": 0.7317837377389272, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.9092875719070435, + "1881223807": 1.9813720285892487, + "1919198188": 1.9560884833335876, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.09564426057040691, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.861636370420456, + "256770874": 1.835601955652237, + "278008524": 1.1011469289660454, + "416183106": 1.812951534986496, + "534717141": 0.0, + "551504635": 1.273692786693573, + "565369392": 1.339533941199382, + "565621826": 0.0, + "585904130": 0.1701989375675718, + "626486813": 0.5741560639580712, + "636680137": 1.9821314811706543, + "706479600": 0.0, + "750448296": 0.1865733889862895, + "75070641": 1.805951327085495, + "767747424": 0.10090384539216757, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.432861328125, + "998761162": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5834752595424653, + "low": 0.6586342200808432, + "mean": 1.1514351414010187, + "n": 5, + "num_samples": 500 + }, + "mean": 1.1514351414010187, + "per_task": { + "LiftPegUpright-v1": 1.4578267857432365, + "PickCube-v1": 1.8952929228544235, + "PullCube-v1": 0.44455755099839095, + "PushCube-v1": 0.748361566892037, + "StackCube-v1": 1.211136880517006 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9684723974298348, + "low": 0.5435695670675014, + "mean": 0.7775791876610247, + "n": 69, + "num_samples": 500 + }, + "n": 69 } }, "thresholds": [ diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_seed.json index 102a0de2db5a28115454b4c687dbc57ca1e0966f..b4aa45a2a5a7e8cd08f81809b18f2dd25b136903 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_seed.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_seed.json @@ -1,331 +1,1083 @@ { "1025740212": { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 1.772506669163704, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17388092747958847, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22749333083629608, + "selected_utility_gain_over_base_at_8": -0.3008803278207779, "selector_regret_at_8": 1.772506669163704, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.772506669163704 }, "1055157354": { + "base_success": 0.0, + "base_utility": 0.19648407399654388, "branch_car_at_8": 0.03480546176433563, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1805709977888097, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22467364370822906, + "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18986818194389343, + "selected_utility_gain_over_base_at_8": -0.006615892052650452, "selector_regret_at_8": 0.03480546176433563, - "support_gap_at_8": 1.775326356291771 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.775326356291771, + "total_car_to_hidden_at_8": 1.8101318180561066 }, "1055567618": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9218642711639404, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19289744165908876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07813572883605957, + "selected_utility_gain_over_base_at_8": -1.9218642711639404, "selector_regret_at_8": 1.9218642711639404, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9218642711639404 }, "1113268940": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.06608206033706665, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.032532089835823584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7025249004364014, + "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6364428400993347, + "selected_utility_gain_over_base_at_8": 0.21413031220436096, "selector_regret_at_8": 0.06608206033706665, - "support_gap_at_8": 1.2974750995635986 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2974750995635986, + "total_car_to_hidden_at_8": 1.3635571599006653 }, "1113371261": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.233029795067053, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1176943004": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17468938592004596, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1231780997": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.5006794333457947, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10207358838016511, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7278411984443665, + "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22716176509857178, + "selected_utility_gain_over_base_at_8": -0.49984437227249146, "selector_regret_at_8": 0.5006794333457947, - "support_gap_at_8": 1.2721588015556335 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2721588015556335, + "total_car_to_hidden_at_8": 1.7728382349014282 }, "1271718007": { + "base_success": 0.5, + "base_utility": 1.415381133556366, "branch_car_at_8": 0.19734752923250198, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2350719661460603, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.2635672688484192, + "proposal_oracle_utility_gain_over_base_at_8": -0.15181386470794678, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.0662197396159172, + "selected_utility_gain_over_base_at_8": -0.34916139394044876, "selector_regret_at_8": 0.19734752923250198, - "support_gap_at_8": 0.7364327311515808 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.7364327311515808, + "total_car_to_hidden_at_8": 0.9337802603840828 }, "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.35906313359737396, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, "selector_regret_at_8": 0.35906313359737396, - "support_gap_at_8": 1.5362297892570496 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5362297892570496, + "total_car_to_hidden_at_8": 1.8952929228544235 }, "1358421325": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 1.90096914768219, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.32948159974174274, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09903085231781006, + "selected_utility_gain_over_base_at_8": -0.7336714267730713, "selector_regret_at_8": 1.90096914768219, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.90096914768219 }, "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002880295505747199, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.008822272371479922, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0008872548933140934, + "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0005992253427393734, + "selected_utility_gain_over_base_at_8": -1.9994007746572606, "selector_regret_at_8": 0.0002880295505747199, - "support_gap_at_8": 0.24256371456431225 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24256371456431225, + "total_car_to_hidden_at_8": 0.24285174411488697 }, "141984615": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, "branch_car_at_8": 0.013830550014972687, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.19434281311769708, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.054234166940053306, + "proposal_oracle_utility_gain_over_base_at_8": -0.039577972143888474, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04040361692508062, + "selected_utility_gain_over_base_at_8": -0.05340852215886116, "selector_regret_at_8": 0.013830550014972687, - "support_gap_at_8": 0.14621367553869882 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.14621367553869882, + "total_car_to_hidden_at_8": 0.1600442255536715 }, "1475440973": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.28389310836792, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22214884767353088, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7161068916320801, + "selected_utility_gain_over_base_at_8": -1.28389310836792, "selector_regret_at_8": 1.28389310836792, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.28389310836792 }, "1483645336": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9686058908700943, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.5995435922806452, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0313941091299057, + "selected_utility_gain_over_base_at_8": -1.9686058908700943, "selector_regret_at_8": 1.9686058908700943, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9686058908700943 }, "1522272686": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.017972871661186218, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.3276128524355476, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1949000507593155, + "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.17692717909812927, + "selected_utility_gain_over_base_at_8": -0.01783391833305359, "selector_regret_at_8": 0.017972871661186218, - "support_gap_at_8": 1.8050999492406845 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8050999492406845, + "total_car_to_hidden_at_8": 1.8230728209018707 }, "1528740417": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.280472755432129, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16399362968800954, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7195272445678711, + "selected_utility_gain_over_base_at_8": -1.280472755432129, "selector_regret_at_8": 1.280472755432129, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.280472755432129 }, "1549324805": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, "branch_car_at_8": 0.01237945475926002, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.23864987006465407, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6833462389186025, + "proposal_oracle_utility_gain_over_base_at_8": -0.6954012094065547, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6709667841593424, + "selected_utility_gain_over_base_at_8": -0.7077806641658148, "selector_regret_at_8": 0.01237945475926002, - "support_gap_at_8": 0.7194042829796672 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7194042829796672, + "total_car_to_hidden_at_8": 0.7317837377389272 }, "1607773860": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.24177237819615016, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1665739511": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2508232858898205, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1747178030": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.034092940390110016, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08595818214364334, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12480536848306656, + "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09071242809295654, + "selected_utility_gain_over_base_at_8": -0.05559416115283966, "selector_regret_at_8": 0.034092940390110016, - "support_gap_at_8": 1.8751946315169334 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8751946315169334, + "total_car_to_hidden_at_8": 1.9092875719070435 }, "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.024854734539985657, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2146835718150399, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.043482705950737, + "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018627971410751343, + "selected_utility_gain_over_base_at_8": -0.013399019837379456, "selector_regret_at_8": 0.024854734539985657, - "support_gap_at_8": 1.956517294049263 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.956517294049263, + "total_car_to_hidden_at_8": 1.9813720285892487 }, "1919198188": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.018839523196220398, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1669540555093941, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06275103986263275, + "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043911516666412354, + "selected_utility_gain_over_base_at_8": 0.0067126452922821045, "selector_regret_at_8": 0.018839523196220398, - "support_gap_at_8": 1.9372489601373672 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9372489601373672, + "total_car_to_hidden_at_8": 1.9560884833335876 }, "1991414747": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11896663871153935, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.14617959762414967, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2015956957": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, "branch_car_at_8": 0.005573729798197746, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.1431252893441326, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.43981083631515505, + "proposal_oracle_utility_gain_over_base_at_8": 0.3726910464465618, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.4342371065169573, + "selected_utility_gain_over_base_at_8": 0.36711731664836406, "selector_regret_at_8": 0.005573729798197746, - "support_gap_at_8": 0.09007053077220917 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.09007053077220917, + "total_car_to_hidden_at_8": 0.09564426057040691 }, "2054899126": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "211245842": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13508468824583414, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1611655950546265, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 1.861636370420456, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09903289408743085, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13836362957954407, + "selected_utility_gain_over_base_at_8": 0.06755638122558594, "selector_regret_at_8": 1.861636370420456, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.861636370420456 }, "256770874": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.046216949820518494, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046630940401554466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.21061499416828156, + "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16439804434776306, + "selected_utility_gain_over_base_at_8": -0.6913500130176544, "selector_regret_at_8": 0.046216949820518494, - "support_gap_at_8": 1.7893850058317184 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7893850058317184, + "total_car_to_hidden_at_8": 1.835601955652237 }, "278008524": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, "branch_car_at_8": 0.020201382460072637, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.15535141304867436, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032865648390725255, + "proposal_oracle_utility_gain_over_base_at_8": -0.0746062200050801, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012664265930652618, + "selected_utility_gain_over_base_at_8": -0.09480760246515274, "selector_regret_at_8": 0.020201382460072637, - "support_gap_at_8": 1.0809455465059727 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.0809455465059727, + "total_car_to_hidden_at_8": 1.1011469289660454 }, "416183106": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.812951534986496, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18072825209943494, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18704846501350403, + "selected_utility_gain_over_base_at_8": -0.013277947902679443, "selector_regret_at_8": 1.812951534986496, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.812951534986496 }, "534717141": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2870871533378466, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "551504635": { + "base_success": 0.0, + "base_utility": 0.7238867878913879, "branch_car_at_8": 1.273692786693573, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2702494805053396, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.726307213306427, + "selected_utility_gain_over_base_at_8": 0.0024204254150390625, "selector_regret_at_8": 1.273692786693573, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.273692786693573 }, "565369392": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, "branch_car_at_8": 0.04557279869914055, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17390197648626904, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.1326704372962316, + "proposal_oracle_utility_gain_over_base_at_8": -0.6670952091614405, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.08709763859709103, + "selected_utility_gain_over_base_at_8": -0.712668007860581, "selector_regret_at_8": 0.04557279869914055, - "support_gap_at_8": 1.2939611425002415 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2939611425002415, + "total_car_to_hidden_at_8": 1.339533941199382 }, "565621826": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "585904130": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, "branch_car_at_8": 0.011032561771571636, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.2625659187813512, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07848061621189117, + "proposal_oracle_utility_gain_over_base_at_8": 0.0031885181864102683, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06744805444031954, + "selected_utility_gain_over_base_at_8": -0.007844043585161367, "selector_regret_at_8": 0.011032561771571636, - "support_gap_at_8": 0.15916637579600015 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.15916637579600015, + "total_car_to_hidden_at_8": 0.1701989375675718 }, "626486813": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, "branch_car_at_8": 0.031722720130346715, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.23561688988950158, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07003506645560265, + "proposal_oracle_utility_gain_over_base_at_8": -0.43540495017077774, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.03831234632525593, + "selected_utility_gain_over_base_at_8": -0.46712767030112445, "selector_regret_at_8": 0.031722720130346715, - "support_gap_at_8": 0.5432130172848701 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5432130172848701, + "total_car_to_hidden_at_8": 0.5741560639580712 }, "636680137": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.006491700187325478, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12019361687075947, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.02436021901667118, + "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.017868518829345703, + "selected_utility_gain_over_base_at_8": -1.9821314811706543, "selector_regret_at_8": 0.006491700187325478, - "support_gap_at_8": 1.9756397809833288 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9756397809833288, + "total_car_to_hidden_at_8": 1.9821314811706543 }, "706479600": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.22468613298280815, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "750448296": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, "branch_car_at_8": 0.01770863775163889, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14149358436045273, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0669428389519453, + "proposal_oracle_utility_gain_over_base_at_8": -0.044590720906853676, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.049234201200306416, + "selected_utility_gain_over_base_at_8": -0.062299358658492565, "selector_regret_at_8": 0.01770863775163889, - "support_gap_at_8": 0.1688647512346506 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1688647512346506, + "total_car_to_hidden_at_8": 0.1865733889862895 }, "75070641": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.3710349500179291, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09051828052202912, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5650836229324341, + "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.194048672914505, + "selected_utility_gain_over_base_at_8": -0.008702963590621948, "selector_regret_at_8": 0.3710349500179291, - "support_gap_at_8": 1.434916377067566 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.434916377067566, + "total_car_to_hidden_at_8": 1.805951327085495 }, "767747424": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, "branch_car_at_8": 0.024658456211909652, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.14269822176062383, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11239026742987335, + "proposal_oracle_utility_gain_over_base_at_8": -0.4351339202839881, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.0877318112179637, + "selected_utility_gain_over_base_at_8": -0.45979237649589777, "selector_regret_at_8": 0.024658456211909652, - "support_gap_at_8": 0.07677002181299031 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07677002181299031, + "total_car_to_hidden_at_8": 0.10090384539216757 }, "77147264": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.12309184879211332, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5784205198287964, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "794181626": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "957101489": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 1.432861328125, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.21219243624964876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.567138671875, + "selected_utility_gain_over_base_at_8": 0.3631737232208252, "selector_regret_at_8": 1.432861328125, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.432861328125 }, "998761162": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.939187914133072, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11706618974004251, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0608120858669281, + "selected_utility_gain_over_base_at_8": -0.007004484534263611, "selector_regret_at_8": 1.939187914133072, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.939187914133072 } } diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_task.json index f5bb06ab396f6516ec8a64866967a12f53b91e13..822efc76f60702db9b35cfb1341b2d3c3e7cec0f 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_task.json +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/metrics_by_task.json @@ -1,37 +1,117 @@ { "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.066929851969083, "branch_car_at_8": 1.029554468890031, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.16081309392707618, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.3333333333333333, + "proposal_oracle_utility_at_8": 1.5717276831467946, + "proposal_oracle_utility_gain_over_base_at_8": 0.5047978311777115, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.5421732142567635, + "selected_utility_gain_over_base_at_8": -0.5247566377123197, "selector_regret_at_8": 1.029554468890031, - "support_gap_at_8": 0.4282723168532054 + "success_selector_gap_at_8": 0.6666666666666666, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.4282723168532054, + "total_car_to_hidden_at_8": 1.4578267857432365 }, "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.35906313359737396, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, "selector_regret_at_8": 0.35906313359737396, - "support_gap_at_8": 1.5362297892570496 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5362297892570496, + "total_car_to_hidden_at_8": 1.8952929228544235 }, "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, "branch_car_at_8": 0.017109284352045506, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, "outcome_ptr_at_8": 0.35294117647058826, "pairwise_causal_calibration_ece": 0.1721079818939321, + "proposal_oracle_success_at_8": 0.14705882352941177, + "proposal_oracle_success_gain_over_base_at_8": -0.14705882352941177, + "proposal_oracle_utility_at_8": 0.3548735701530377, + "proposal_oracle_utility_gain_over_base_at_8": -0.29635635063031635, + "selected_success_at_8": 0.14705882352941177, + "selected_success_gain_over_base_at_8": -0.14705882352941177, + "selected_utility_at_8": 0.3377642858009922, + "selected_utility_gain_over_base_at_8": -0.31346563498236185, "selector_regret_at_8": 0.017109284352045506, - "support_gap_at_8": 0.4276017144216252 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.17647058823529413, + "success_total_car_to_hidden_at_8": 0.17647058823529413, + "support_gap_at_8": 0.4276017144216252, + "total_car_to_hidden_at_8": 0.44455755099839095 }, "PushCube-v1": { + "base_success": 0.38461538461538464, + "base_utility": 0.8217729788560134, "branch_car_at_8": 0.4485462617415648, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6153846153846154, "pairwise_causal_calibration_ece": 0.15912329792465574, + "proposal_oracle_success_at_8": 0.8461538461538461, + "proposal_oracle_success_gain_over_base_at_8": 0.46153846153846156, + "proposal_oracle_utility_at_8": 1.7001846948495278, + "proposal_oracle_utility_gain_over_base_at_8": 0.8784117159935144, + "selected_success_at_8": 0.6153846153846154, + "selected_success_gain_over_base_at_8": 0.23076923076923078, + "selected_utility_at_8": 1.251638433107963, + "selected_utility_gain_over_base_at_8": 0.4298654542519496, "selector_regret_at_8": 0.4485462617415648, - "support_gap_at_8": 0.2998153051504722 + "success_selector_gap_at_8": 0.23076923076923078, + "success_support_gap_at_8": 0.15384615384615385, + "success_total_car_to_hidden_at_8": 0.38461538461538464, + "support_gap_at_8": 0.2998153051504722, + "total_car_to_hidden_at_8": 0.748361566892037 }, "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8047854532798131, "branch_car_at_8": 0.5301474064588547, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.20081454850953362, + "proposal_oracle_success_at_8": 0.6, + "proposal_oracle_success_gain_over_base_at_8": 0.4, + "proposal_oracle_utility_at_8": 1.3190105259418488, + "proposal_oracle_utility_gain_over_base_at_8": 0.5142250726620357, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.13333333333333333, + "selected_utility_at_8": 0.788863119482994, + "selected_utility_gain_over_base_at_8": -0.015922333796819052, "selector_regret_at_8": 0.5301474064588547, - "support_gap_at_8": 0.6809894740581512 + "success_selector_gap_at_8": 0.26666666666666666, + "success_support_gap_at_8": 0.4, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.6809894740581512, + "total_car_to_hidden_at_8": 1.211136880517006 } } diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/report.md b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/report.md index 98aa63029ba52a7a1859368dee5ffbe347b83d79..f59b5289d284e2f4db73b837b1ca29c9be2896e0 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/report.md +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/report.md @@ -4,8 +4,24 @@ K: `8` | Metric | N | Micro mean | 95% CI | Task macro | Seed macro | | --- | ---: | ---: | ---: | ---: | ---: | +| base_success | 69 | 0.2899 | [0.1884, 0.4203] | 0.2424 | 0.3298 | +| base_utility | 69 | 0.7457 | [0.5696, 0.9620] | 0.6996 | 0.8569 | | branch_car_at_8 | 69 | 0.3029 | [0.1642, 0.4617] | 0.4769 | 0.4321 | +| hidden_chart_oracle_success_at_8 | 69 | 0.6667 | [0.5507, 0.7826] | 0.8647 | 0.8358 | +| hidden_chart_oracle_utility_at_8 | 69 | 1.4000 | [1.1855, 1.6062] | 1.7565 | 1.7062 | | outcome_ptr_at_8 | 69 | 0.5072 | [0.3623, 0.6087] | 0.6603 | 0.5340 | | pairwise_causal_calibration_ece | 69 | 0.1765 | [0.1491, 0.2027] | 0.1944 | 0.1718 | +| proposal_oracle_success_at_8 | 69 | 0.4203 | [0.3188, 0.5507] | 0.4520 | 0.5539 | +| proposal_oracle_success_gain_over_base_at_8 | 69 | 0.1304 | [-0.0145, 0.2609] | 0.2096 | 0.2241 | +| proposal_oracle_utility_at_8 | 69 | 0.9253 | [0.7270, 1.1546] | 1.0819 | 1.1974 | +| proposal_oracle_utility_gain_over_base_at_8 | 69 | 0.1796 | [-0.0557, 0.4002] | 0.3823 | 0.3406 | +| selected_success_at_8 | 69 | 0.2609 | [0.1739, 0.3623] | 0.2192 | 0.3199 | +| selected_success_gain_over_base_at_8 | 69 | -0.0290 | [-0.1594, 0.0870] | -0.0233 | -0.0099 | +| selected_utility_at_8 | 69 | 0.6224 | [0.4473, 0.8056] | 0.6050 | 0.7653 | +| selected_utility_gain_over_base_at_8 | 69 | -0.1233 | [-0.3535, 0.0836] | -0.0946 | -0.0915 | | selector_regret_at_8 | 69 | 0.3029 | [0.1642, 0.4617] | 0.4769 | 0.4321 | +| success_selector_gap_at_8 | 69 | 0.1594 | [0.0870, 0.2609] | 0.2328 | 0.2340 | +| success_support_gap_at_8 | 69 | 0.2464 | [0.1449, 0.3478] | 0.4127 | 0.2819 | +| success_total_car_to_hidden_at_8 | 69 | 0.4058 | [0.2754, 0.5217] | 0.6456 | 0.5160 | | support_gap_at_8 | 69 | 0.4747 | [0.3004, 0.6486] | 0.6746 | 0.5088 | +| total_car_to_hidden_at_8 | 69 | 0.7776 | [0.5436, 0.9685] | 1.1514 | 0.9409 | diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/table.tex b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/table.tex index 1ff0216d75cdcfaf6392effc32c22016aee61706..642f00b60f88d783018f8f64fec0cbc57c6379fa 100644 --- a/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/table.tex +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/measured_metrics/table.tex @@ -3,10 +3,26 @@ \toprule Metric & N & Micro mean & CI low & CI high \\ \midrule +base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\ +base\_utility & 69 & 0.7457 & 0.5696 & 0.9620 \\ branch\_car\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\ +hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\ outcome\_ptr\_at\_8 & 69 & 0.5072 & 0.3623 & 0.6087 \\ pairwise\_causal\_calibration\_ece & 69 & 0.1765 & 0.1491 & 0.2027 \\ +proposal\_oracle\_success\_at\_8 & 69 & 0.4203 & 0.3188 & 0.5507 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1304 & -0.0145 & 0.2609 \\ +proposal\_oracle\_utility\_at\_8 & 69 & 0.9253 & 0.7270 & 1.1546 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.1796 & -0.0557 & 0.4002 \\ +selected\_success\_at\_8 & 69 & 0.2609 & 0.1739 & 0.3623 \\ +selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.0290 & -0.1594 & 0.0870 \\ +selected\_utility\_at\_8 & 69 & 0.6224 & 0.4473 & 0.8056 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.1233 & -0.3535 & 0.0836 \\ selector\_regret\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\ +success\_selector\_gap\_at\_8 & 69 & 0.1594 & 0.0870 & 0.2609 \\ +success\_support\_gap\_at\_8 & 69 & 0.2464 & 0.1449 & 0.3478 \\ +success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\ support\_gap\_at\_8 & 69 & 0.4747 & 0.3004 & 0.6486 \\ +total\_car\_to\_hidden\_at\_8 & 69 & 0.7776 & 0.5436 & 0.9685 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/metrics.json b/workspace/runs/ctt_residual_rollout_val69_seed2/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..455406329f9d1a2c11df6cb3bfe791af995fb64d --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/metrics.json @@ -0,0 +1,1918 @@ +{ + "checkpoint": "runs/ctt_residual_full_seed2/model.pt", + "k": 8, + "measured_metric_summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4521276595744681, + "low": 0.20921985815602834, + "mean": 0.3297872340425532, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3297872340425532, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 1.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 1.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.3333333333333333, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.25, + "77147264": 0.5, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.33815987933634994, + "low": 0.10666666666666666, + "mean": 0.24241327300150828, + "n": 5, + "num_samples": 500 + }, + "mean": 0.24241327300150828, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.42028985507246375, + "low": 0.18840579710144928, + "mean": 0.2898550724637681, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.0924235453427908, + "low": 0.6323911708297458, + "mean": 0.8568741302496377, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8568741302496377, + "per_seed": { + "1025740212": 0.528373658657074, + "1055157354": 0.19648407399654388, + "1055567618": 2.0, + "1113268940": 0.42231252789497375, + "1113371261": 2.0, + "1176943004": 0.05094052851200104, + "1189392917": 0.10841864347457886, + "1231780997": 0.7270061373710632, + "1271718007": 1.415381133556366, + "1273247072": 0.15342965722084045, + "1358421325": 0.8327022790908813, + "1375492786": 2.0, + "141984615": 0.09381213908394177, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.19476109743118286, + "1528740417": 2.0, + "1549324805": 1.3787474483251572, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.1463065892457962, + "1881223807": 0.0320269912481308, + "1919198188": 0.03719887137413025, + "1991414747": 2.0, + "1997090716": 0.13927479088306427, + "2015956957": 0.06711978986859321, + "2054899126": 2.0, + "211245842": 0.8388344049453735, + "2141802816": 0.07080724835395813, + "256770874": 0.8557480573654175, + "278008524": 0.10747186839580536, + "416183106": 0.20032641291618347, + "534717141": 2.0, + "551504635": 0.7238867878913879, + "565369392": 0.7997656464576721, + "565621826": 2.0, + "585904130": 0.07529209802548091, + "626486813": 0.5054400166263804, + "636680137": 2.0, + "706479600": 0.8350887894630432, + "750448296": 0.11153355985879898, + "75070641": 0.20275163650512695, + "767747424": 0.5475241877138615, + "77147264": 1.4215794801712036, + "794181626": 0.18095605075359344, + "957101489": 0.2039649486541748, + "998761162": 0.06781657040119171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9313609859940832, + "low": 0.4139719756444295, + "mean": 0.6996295724218208, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6996295724218208, + "per_task": { + "LiftPegUpright-v1": 1.066929851969083, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.8217729788560134, + "StackCube-v1": 0.8047854532798131 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.9619567723796312, + "low": 0.5695875146345276, + "mean": 0.7456761826086196, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "branch_car_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6595998992103133, + "low": 0.2230010245754945, + "mean": 0.43210832736758736, + "n": 47, + "num_samples": 500 + }, + "mean": 0.43210832736758736, + "per_seed": { + "1025740212": 1.772506669163704, + "1055157354": 0.03480546176433563, + "1055567618": 1.9218642711639404, + "1113268940": 0.06608206033706665, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.5006794333457947, + "1271718007": 0.19734752923250198, + "1273247072": 0.35906313359737396, + "1358421325": 1.90096914768219, + "1375492786": 0.0002880295505747199, + "141984615": 0.013830550014972687, + "1475440973": 1.28389310836792, + "1483645336": 1.9686058908700943, + "1522272686": 0.017972871661186218, + "1528740417": 1.280472755432129, + "1549324805": 0.01237945475926002, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.034092940390110016, + "1881223807": 0.024854734539985657, + "1919198188": 0.018839523196220398, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.005573729798197746, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.861636370420456, + "256770874": 0.046216949820518494, + "278008524": 0.020201382460072637, + "416183106": 1.812951534986496, + "534717141": 0.0, + "551504635": 1.273692786693573, + "565369392": 0.04557279869914055, + "565621826": 0.0, + "585904130": 0.011032561771571636, + "626486813": 0.031722720130346715, + "636680137": 0.006491700187325478, + "706479600": 0.0, + "750448296": 0.01770863775163889, + "75070641": 0.3710349500179291, + "767747424": 0.024658456211909652, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.432861328125, + "998761162": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7792545604018064, + "low": 0.18968407530785322, + "mean": 0.476884111007974, + "n": 5, + "num_samples": 500 + }, + "mean": 0.476884111007974, + "per_task": { + "LiftPegUpright-v1": 1.029554468890031, + "PickCube-v1": 0.35906313359737396, + "PullCube-v1": 0.017109284352045506, + "PushCube-v1": 0.4485462617415648, + "StackCube-v1": 0.5301474064588547 + } + }, + "metric": "branch_car_at_8", + "micro": { + "confidence": 0.95, + "high": 0.46172967990117986, + "low": 0.1641734795007126, + "mean": 0.3029191031076851, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.922695035460993, + "low": 0.7351063829787235, + "mean": 0.8358156028368794, + "n": 47, + "num_samples": 500 + }, + "mean": 0.8358156028368794, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 1.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.6666666666666666, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.782608695652174, + "low": 0.5507246376811594, + "mean": 0.6666666666666666, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8615907955016442, + "low": 1.529367521088174, + "mean": 1.7062028111638965, + "n": 47, + "num_samples": 500 + }, + "mean": 1.7062028111638965, + "per_seed": { + "1025740212": 2.0, + "1055157354": 2.0, + "1055567618": 2.0, + "1113268940": 2.0, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 2.0, + "1271718007": 2.0, + "1273247072": 2.0, + "1358421325": 2.0, + "1375492786": 0.24345096945762634, + "141984615": 0.20044784247875214, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 2.0, + "1528740417": 2.0, + "1549324805": 1.4027505218982697, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 2.0, + "1881223807": 2.0, + "1919198188": 2.0, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.5298813670873642, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 2.0, + "278008524": 1.113811194896698, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 1.4266315797964733, + "565621826": 2.0, + "585904130": 0.23764699200789133, + "626486813": 0.6124684102833271, + "636680137": 2.0, + "706479600": 2.0, + "750448296": 0.23580759018659592, + "75070641": 2.0, + "767747424": 0.18863565661013126, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.6062410234109215, + "low": 1.1855228276572365, + "mean": 1.3999846732054932, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "outcome_ptr_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6567375886524823, + "low": 0.3964539007092199, + "mean": 0.5340425531914894, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5340425531914894, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 1.0, + "1271718007": 0.0, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.3333333333333333, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.6, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.6666666666666666, + "626486813": 0.75, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.75, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.8564102564102564, + "low": 0.46817496229260935, + "mean": 0.6603318250377074, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6603318250377074, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 1.0, + "PullCube-v1": 0.35294117647058826, + "PushCube-v1": 0.6153846153846154, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "outcome_ptr_at_8", + "micro": { + "confidence": 0.95, + "high": 0.6086956521739131, + "low": 0.36231884057971014, + "mean": 0.5072463768115942, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "pairwise_causal_calibration_ece": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.20239102672265116, + "low": 0.14332855009235143, + "mean": 0.17181636350182927, + "n": 47, + "num_samples": 500 + }, + "mean": 0.17181636350182927, + "per_seed": { + "1025740212": 0.17388092747958847, + "1055157354": 0.1805709977888097, + "1055567618": 0.19289744165908876, + "1113268940": 0.032532089835823584, + "1113371261": 0.233029795067053, + "1176943004": 0.17468938592004596, + "1189392917": 0.0, + "1231780997": 0.10207358838016511, + "1271718007": 0.2350719661460603, + "1273247072": 0.2793744975404514, + "1358421325": 0.32948159974174274, + "1375492786": 0.008822272371479922, + "141984615": 0.19434281311769708, + "1475440973": 0.22214884767353088, + "1483645336": 0.5995435922806452, + "1522272686": 0.3276128524355476, + "1528740417": 0.16399362968800954, + "1549324805": 0.23864987006465407, + "1607773860": 0.24177237819615016, + "1665739511": 0.2508232858898205, + "1747178030": 0.08595818214364334, + "1881223807": 0.2146835718150399, + "1919198188": 0.1669540555093941, + "1991414747": 0.11896663871153935, + "1997090716": 0.14617959762414967, + "2015956957": 0.1431252893441326, + "2054899126": 0.0, + "211245842": 0.13508468824583414, + "2141802816": 0.09903289408743085, + "256770874": 0.046630940401554466, + "278008524": 0.15535141304867436, + "416183106": 0.18072825209943494, + "534717141": 0.2870871533378466, + "551504635": 0.2702494805053396, + "565369392": 0.17390197648626904, + "565621826": 0.0, + "585904130": 0.2625659187813512, + "626486813": 0.23561688988950158, + "636680137": 0.12019361687075947, + "706479600": 0.22468613298280815, + "750448296": 0.14149358436045273, + "75070641": 0.09051828052202912, + "767747424": 0.14269822176062383, + "77147264": 0.12309184879211332, + "794181626": 0.0, + "957101489": 0.21219243624964876, + "998761162": 0.11706618974004251 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2399502270115928, + "low": 0.16431717151236627, + "mean": 0.1944466839591298, + "n": 5, + "num_samples": 500 + }, + "mean": 0.1944466839591298, + "per_task": { + "LiftPegUpright-v1": 0.16081309392707618, + "PickCube-v1": 0.2793744975404514, + "PullCube-v1": 0.1721079818939321, + "PushCube-v1": 0.15912329792465574, + "StackCube-v1": 0.20081454850953362 + } + }, + "metric": "pairwise_causal_calibration_ece", + "micro": { + "confidence": 0.95, + "high": 0.2026857252748648, + "low": 0.1491416444266403, + "mean": 0.17647457313275544, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6808510638297872, + "low": 0.425531914893617, + "mean": 0.5539007092198581, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5539007092198581, + "per_seed": { + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 1.0, + "551504635": 1.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7117948717948719, + "low": 0.14941176470588236, + "mean": 0.4519758672699849, + "n": 5, + "num_samples": 500 + }, + "mean": 0.4519758672699849, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.14705882352941177, + "PushCube-v1": 0.8461538461538461, + "StackCube-v1": 0.6 + } + }, + "metric": "proposal_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5507246376811594, + "low": 0.3188405797101449, + "mean": 0.42028985507246375, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.35496453900709213, + "low": 0.08120567375886525, + "mean": 0.224113475177305, + "n": 47, + "num_samples": 500 + }, + "mean": 0.224113475177305, + "per_seed": { + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 1.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": -0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4235897435897436, + "low": -0.0215686274509804, + "mean": 0.20956259426847662, + "n": 5, + "num_samples": 500 + }, + "mean": 0.20956259426847662, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.14705882352941177, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.4 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2608695652173913, + "low": -0.014492753623188406, + "mean": 0.13043478260869565, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.4388152989261644, + "low": 0.9575430694977415, + "mean": 1.1974476296841894, + "n": 47, + "num_samples": 500 + }, + "mean": 1.1974476296841894, + "per_seed": { + "1025740212": 2.0, + "1055157354": 0.22467364370822906, + "1055567618": 2.0, + "1113268940": 0.7025249004364014, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.7278411984443665, + "1271718007": 1.2635672688484192, + "1273247072": 0.46377021074295044, + "1358421325": 2.0, + "1375492786": 0.0008872548933140934, + "141984615": 0.054234166940053306, + "1475440973": 2.0, + "1483645336": 2.0, + "1522272686": 0.1949000507593155, + "1528740417": 2.0, + "1549324805": 0.6833462389186025, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.12480536848306656, + "1881223807": 0.043482705950737, + "1919198188": 0.06275103986263275, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.43981083631515505, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 2.0, + "256770874": 0.21061499416828156, + "278008524": 0.032865648390725255, + "416183106": 2.0, + "534717141": 2.0, + "551504635": 2.0, + "565369392": 0.1326704372962316, + "565621826": 2.0, + "585904130": 0.07848061621189117, + "626486813": 0.07003506645560265, + "636680137": 0.02436021901667118, + "706479600": 2.0, + "750448296": 0.0669428389519453, + "75070641": 0.5650836229324341, + "767747424": 0.11239026742987335, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 2.0, + "998761162": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5220236249459096, + "low": 0.5982443927517891, + "mean": 1.081913336966832, + "n": 5, + "num_samples": 500 + }, + "mean": 1.081913336966832, + "per_task": { + "LiftPegUpright-v1": 1.5717276831467946, + "PickCube-v1": 0.46377021074295044, + "PullCube-v1": 0.3548735701530377, + "PushCube-v1": 1.7001846948495278, + "StackCube-v1": 1.3190105259418488 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.1545684401146676, + "low": 0.7270322287424832, + "mean": 0.9253245886521535, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5855443862486458, + "low": 0.08830777057388746, + "mean": 0.3405734994345517, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3405734994345517, + "per_seed": { + "1025740212": 1.471626341342926, + "1055157354": 0.02818956971168518, + "1055567618": 0.0, + "1113268940": 0.2802123725414276, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": 0.0008350610733032227, + "1271718007": -0.15181386470794678, + "1273247072": 0.31034055352211, + "1358421325": 1.1672977209091187, + "1375492786": -1.999112745106686, + "141984615": -0.039577972143888474, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0001389533281326294, + "1528740417": 0.0, + "1549324805": -0.6954012094065547, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": -0.021501220762729645, + "1881223807": 0.011455714702606201, + "1919198188": 0.025552168488502502, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": 0.3726910464465618, + "2054899126": 0.0, + "211245842": 1.1611655950546265, + "2141802816": 1.9291927516460419, + "256770874": -0.6451330631971359, + "278008524": -0.0746062200050801, + "416183106": 1.7996735870838165, + "534717141": 0.0, + "551504635": 1.276113212108612, + "565369392": -0.6670952091614405, + "565621826": 0.0, + "585904130": 0.0031885181864102683, + "626486813": -0.43540495017077774, + "636680137": -1.9756397809833288, + "706479600": 1.1649112105369568, + "750448296": -0.044590720906853676, + "75070641": 0.36233198642730713, + "767747424": -0.4351339202839881, + "77147264": 0.5784205198287964, + "794181626": 1.8190439492464066, + "957101489": 1.7960350513458252, + "998761162": 1.9321834295988083 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.6900747065360731, + "low": -0.01290068514136069, + "mean": 0.382283764545011, + "n": 5, + "num_samples": 500 + }, + "mean": 0.382283764545011, + "per_task": { + "LiftPegUpright-v1": 0.5047978311777115, + "PickCube-v1": 0.31034055352211, + "PullCube-v1": -0.29635635063031635, + "PushCube-v1": 0.8784117159935144, + "StackCube-v1": 0.5142250726620357 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.40019721312421386, + "low": -0.055687820795329586, + "mean": 0.179648406043534, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4482269503546099, + "low": 0.19148936170212766, + "mean": 0.3198581560283688, + "n": 47, + "num_samples": 500 + }, + "mean": 0.3198581560283688, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": 0.0, + "1113268940": 0.0, + "1113371261": 1.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.5, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 0.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 1.0, + "1665739511": 1.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 1.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 1.0, + "211245842": 1.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 1.0, + "551504635": 0.0, + "565369392": 0.0, + "565621826": 1.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 1.0, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.43589743589743596, + "low": 0.058823529411764705, + "mean": 0.21915535444947212, + "n": 5, + "num_samples": 500 + }, + "mean": 0.21915535444947212, + "per_task": { + "LiftPegUpright-v1": 0.0, + "PickCube-v1": 0.0, + "PullCube-v1": 0.14705882352941177, + "PushCube-v1": 0.6153846153846154, + "StackCube-v1": 0.3333333333333333 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.36231884057971014, + "low": 0.17391304347826086, + "mean": 0.2608695652173913, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.11950354609929079, + "low": -0.16241134751773048, + "mean": -0.0099290780141844, + "n": 47, + "num_samples": 500 + }, + "mean": -0.0099290780141844, + "per_seed": { + "1025740212": 0.0, + "1055157354": 0.0, + "1055567618": -1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 1.0, + "1189392917": 1.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 0.0, + "1375492786": -1.0, + "141984615": 0.0, + "1475440973": -1.0, + "1483645336": -1.0, + "1522272686": 0.0, + "1528740417": -1.0, + "1549324805": -0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 1.0, + "2015956957": 0.2, + "2054899126": 0.0, + "211245842": 1.0, + "2141802816": 0.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": -0.3333333333333333, + "565621826": 0.0, + "585904130": 0.0, + "626486813": -0.25, + "636680137": -1.0, + "706479600": 1.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": -0.25, + "77147264": 0.5, + "794181626": 1.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.14564102564102566, + "low": -0.20274509803921567, + "mean": -0.02325791855203619, + "n": 5, + "num_samples": 500 + }, + "mean": -0.02325791855203619, + "per_task": { + "LiftPegUpright-v1": -0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": -0.14705882352941177, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.13333333333333333 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.08695652173913043, + "low": -0.15942028985507245, + "mean": -0.028985507246376812, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9970987782217497, + "low": 0.5411203128707609, + "mean": 0.7653393023166019, + "n": 47, + "num_samples": 500 + }, + "mean": 0.7653393023166019, + "per_seed": { + "1025740212": 0.22749333083629608, + "1055157354": 0.18986818194389343, + "1055567618": 0.07813572883605957, + "1113268940": 0.6364428400993347, + "1113371261": 2.0, + "1176943004": 2.0, + "1189392917": 2.0, + "1231780997": 0.22716176509857178, + "1271718007": 1.0662197396159172, + "1273247072": 0.10470707714557648, + "1358421325": 0.09903085231781006, + "1375492786": 0.0005992253427393734, + "141984615": 0.04040361692508062, + "1475440973": 0.7161068916320801, + "1483645336": 0.0313941091299057, + "1522272686": 0.17692717909812927, + "1528740417": 0.7195272445678711, + "1549324805": 0.6709667841593424, + "1607773860": 2.0, + "1665739511": 2.0, + "1747178030": 0.09071242809295654, + "1881223807": 0.018627971410751343, + "1919198188": 0.043911516666412354, + "1991414747": 2.0, + "1997090716": 2.0, + "2015956957": 0.4342371065169573, + "2054899126": 2.0, + "211245842": 2.0, + "2141802816": 0.13836362957954407, + "256770874": 0.16439804434776306, + "278008524": 0.012664265930652618, + "416183106": 0.18704846501350403, + "534717141": 2.0, + "551504635": 0.726307213306427, + "565369392": 0.08709763859709103, + "565621826": 2.0, + "585904130": 0.06744805444031954, + "626486813": 0.03831234632525593, + "636680137": 0.017868518829345703, + "706479600": 2.0, + "750448296": 0.049234201200306416, + "75070641": 0.194048672914505, + "767747424": 0.0877318112179637, + "77147264": 2.0, + "794181626": 2.0, + "957101489": 0.567138671875, + "998761162": 0.0608120858669281 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9739732449329817, + "low": 0.2881497273441431, + "mean": 0.6050292259588579, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6050292259588579, + "per_task": { + "LiftPegUpright-v1": 0.5421732142567635, + "PickCube-v1": 0.10470707714557648, + "PullCube-v1": 0.3377642858009922, + "PushCube-v1": 1.251638433107963, + "StackCube-v1": 0.788863119482994 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.8055619765385189, + "low": 0.44728328000781115, + "mean": 0.6224054855444685, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.13649843921744537, + "low": -0.349154780383126, + "mean": -0.09153482793303577, + "n": 47, + "num_samples": 500 + }, + "mean": -0.09153482793303577, + "per_seed": { + "1025740212": -0.3008803278207779, + "1055157354": -0.006615892052650452, + "1055567618": -1.9218642711639404, + "1113268940": 0.21413031220436096, + "1113371261": 0.0, + "1176943004": 1.949059471487999, + "1189392917": 1.8915813565254211, + "1231780997": -0.49984437227249146, + "1271718007": -0.34916139394044876, + "1273247072": -0.04872258007526398, + "1358421325": -0.7336714267730713, + "1375492786": -1.9994007746572606, + "141984615": -0.05340852215886116, + "1475440973": -1.28389310836792, + "1483645336": -1.9686058908700943, + "1522272686": -0.01783391833305359, + "1528740417": -1.280472755432129, + "1549324805": -0.7077806641658148, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": -0.05559416115283966, + "1881223807": -0.013399019837379456, + "1919198188": 0.0067126452922821045, + "1991414747": 0.0, + "1997090716": 1.8607252091169357, + "2015956957": 0.36711731664836406, + "2054899126": 0.0, + "211245842": 1.1611655950546265, + "2141802816": 0.06755638122558594, + "256770874": -0.6913500130176544, + "278008524": -0.09480760246515274, + "416183106": -0.013277947902679443, + "534717141": 0.0, + "551504635": 0.0024204254150390625, + "565369392": -0.712668007860581, + "565621826": 0.0, + "585904130": -0.007844043585161367, + "626486813": -0.46712767030112445, + "636680137": -1.9821314811706543, + "706479600": 1.1649112105369568, + "750448296": -0.062299358658492565, + "75070641": -0.008702963590621948, + "767747424": -0.45979237649589777, + "77147264": 0.5784205198287964, + "794181626": 1.8190439492464066, + "957101489": 0.3631737232208252, + "998761162": -0.007004484534263611 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.1623927814226884, + "low": -0.3557238355283534, + "mean": -0.09460034646296299, + "n": 5, + "num_samples": 500 + }, + "mean": -0.09460034646296299, + "per_task": { + "LiftPegUpright-v1": -0.5247566377123197, + "PickCube-v1": -0.04872258007526398, + "PullCube-v1": -0.31346563498236185, + "PushCube-v1": 0.4298654542519496, + "StackCube-v1": -0.015922333796819052 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.08361495469791302, + "low": -0.3535496463047583, + "mean": -0.1232706970641511, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6595998992103133, + "low": 0.2230010245754945, + "mean": 0.43210832736758736, + "n": 47, + "num_samples": 500 + }, + "mean": 0.43210832736758736, + "per_seed": { + "1025740212": 1.772506669163704, + "1055157354": 0.03480546176433563, + "1055567618": 1.9218642711639404, + "1113268940": 0.06608206033706665, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.5006794333457947, + "1271718007": 0.19734752923250198, + "1273247072": 0.35906313359737396, + "1358421325": 1.90096914768219, + "1375492786": 0.0002880295505747199, + "141984615": 0.013830550014972687, + "1475440973": 1.28389310836792, + "1483645336": 1.9686058908700943, + "1522272686": 0.017972871661186218, + "1528740417": 1.280472755432129, + "1549324805": 0.01237945475926002, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.034092940390110016, + "1881223807": 0.024854734539985657, + "1919198188": 0.018839523196220398, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.005573729798197746, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.861636370420456, + "256770874": 0.046216949820518494, + "278008524": 0.020201382460072637, + "416183106": 1.812951534986496, + "534717141": 0.0, + "551504635": 1.273692786693573, + "565369392": 0.04557279869914055, + "565621826": 0.0, + "585904130": 0.011032561771571636, + "626486813": 0.031722720130346715, + "636680137": 0.006491700187325478, + "706479600": 0.0, + "750448296": 0.01770863775163889, + "75070641": 0.3710349500179291, + "767747424": 0.024658456211909652, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.432861328125, + "998761162": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7792545604018064, + "low": 0.18968407530785322, + "mean": 0.476884111007974, + "n": 5, + "num_samples": 500 + }, + "mean": 0.476884111007974, + "per_task": { + "LiftPegUpright-v1": 1.029554468890031, + "PickCube-v1": 0.35906313359737396, + "PullCube-v1": 0.017109284352045506, + "PushCube-v1": 0.4485462617415648, + "StackCube-v1": 0.5301474064588547 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.46172967990117986, + "low": 0.1641734795007126, + "mean": 0.3029191031076851, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3617021276595745, + "low": 0.10638297872340426, + "mean": 0.23404255319148937, + "n": 47, + "num_samples": 500 + }, + "mean": 0.23404255319148937, + "per_seed": { + "1025740212": 1.0, + "1055157354": 0.0, + "1055567618": 1.0, + "1113268940": 0.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 0.0, + "1271718007": 0.0, + "1273247072": 0.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 0.0, + "1528740417": 1.0, + "1549324805": 0.0, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 0.0, + "1881223807": 0.0, + "1919198188": 0.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.0, + "256770874": 0.0, + "278008524": 0.0, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.0, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.0, + "636680137": 0.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 0.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.4461538461538462, + "low": 0.046153846153846156, + "mean": 0.2328205128205128, + "n": 5, + "num_samples": 500 + }, + "mean": 0.2328205128205128, + "per_task": { + "LiftPegUpright-v1": 0.6666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.26666666666666666 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2608695652173913, + "low": 0.08695652173913043, + "mean": 0.15942028985507245, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.4148936170212766, + "low": 0.1595744680851064, + "mean": 0.2819148936170212, + "n": 47, + "num_samples": 500 + }, + "mean": 0.2819148936170212, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.0, + "1055567618": 0.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 0.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.0, + "1528740417": 0.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7152941176470589, + "low": 0.19879336349924587, + "mean": 0.4127300150829562, + "n": 5, + "num_samples": 500 + }, + "mean": 0.4127300150829562, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 1.0, + "PullCube-v1": 0.17647058823529413, + "PushCube-v1": 0.15384615384615385, + "StackCube-v1": 0.4 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.34782608695652173, + "low": 0.14492753623188406, + "mean": 0.2463768115942029, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6524822695035459, + "low": 0.3776595744680851, + "mean": 0.5159574468085107, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5159574468085107, + "per_seed": { + "1025740212": 1.0, + "1055157354": 1.0, + "1055567618": 1.0, + "1113268940": 1.0, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.0, + "1271718007": 0.5, + "1273247072": 1.0, + "1358421325": 1.0, + "1375492786": 0.0, + "141984615": 0.0, + "1475440973": 1.0, + "1483645336": 1.0, + "1522272686": 1.0, + "1528740417": 1.0, + "1549324805": 0.3333333333333333, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.0, + "1881223807": 1.0, + "1919198188": 1.0, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.0, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.0, + "256770874": 1.0, + "278008524": 0.5, + "416183106": 1.0, + "534717141": 0.0, + "551504635": 1.0, + "565369392": 0.6666666666666666, + "565621826": 0.0, + "585904130": 0.0, + "626486813": 0.25, + "636680137": 1.0, + "706479600": 0.0, + "750448296": 0.0, + "75070641": 1.0, + "767747424": 0.0, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.0, + "998761162": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9333333333333332, + "low": 0.3161387631975867, + "mean": 0.645550527903469, + "n": 5, + "num_samples": 500 + }, + "mean": 0.645550527903469, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.17647058823529413, + "PushCube-v1": 0.38461538461538464, + "StackCube-v1": 0.6666666666666666 + } + }, + "metric": "success_total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.5217391304347826, + "low": 0.2753623188405797, + "mean": 0.4057971014492754, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.7286775349375151, + "low": 0.30572481987938444, + "mean": 0.5087829326731087, + "n": 47, + "num_samples": 500 + }, + "mean": 0.5087829326731087, + "per_seed": { + "1025740212": 0.0, + "1055157354": 1.775326356291771, + "1055567618": 0.0, + "1113268940": 1.2974750995635986, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.2721588015556335, + "1271718007": 0.7364327311515808, + "1273247072": 1.5362297892570496, + "1358421325": 0.0, + "1375492786": 0.24256371456431225, + "141984615": 0.14621367553869882, + "1475440973": 0.0, + "1483645336": 0.0, + "1522272686": 1.8050999492406845, + "1528740417": 0.0, + "1549324805": 0.7194042829796672, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.8751946315169334, + "1881223807": 1.956517294049263, + "1919198188": 1.9372489601373672, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.09007053077220917, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 0.0, + "256770874": 1.7893850058317184, + "278008524": 1.0809455465059727, + "416183106": 0.0, + "534717141": 0.0, + "551504635": 0.0, + "565369392": 1.2939611425002415, + "565621826": 0.0, + "585904130": 0.15916637579600015, + "626486813": 0.5432130172848701, + "636680137": 1.9756397809833288, + "706479600": 0.0, + "750448296": 0.1688647512346506, + "75070641": 1.434916377067566, + "767747424": 0.07677002181299031, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 0.0, + "998761162": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.143456111250185, + "low": 0.37662127119948, + "mean": 0.6745817199481008, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6745817199481008, + "per_task": { + "LiftPegUpright-v1": 0.4282723168532054, + "PickCube-v1": 1.5362297892570496, + "PullCube-v1": 0.4276017144216252, + "PushCube-v1": 0.2998153051504722, + "StackCube-v1": 0.6809894740581512 + } + }, + "metric": "support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.648598097173659, + "low": 0.30040934245672374, + "mean": 0.4747356965005789, + "n": 69, + "num_samples": 500 + }, + "n": 69 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1787349130094698, + "low": 0.7077262575365768, + "mean": 0.9408635088472943, + "n": 47, + "num_samples": 500 + }, + "mean": 0.9408635088472943, + "per_seed": { + "1025740212": 1.772506669163704, + "1055157354": 1.8101318180561066, + "1055567618": 1.9218642711639404, + "1113268940": 1.3635571599006653, + "1113371261": 0.0, + "1176943004": 0.0, + "1189392917": 0.0, + "1231780997": 1.7728382349014282, + "1271718007": 0.9337802603840828, + "1273247072": 1.8952929228544235, + "1358421325": 1.90096914768219, + "1375492786": 0.24285174411488697, + "141984615": 0.1600442255536715, + "1475440973": 1.28389310836792, + "1483645336": 1.9686058908700943, + "1522272686": 1.8230728209018707, + "1528740417": 1.280472755432129, + "1549324805": 0.7317837377389272, + "1607773860": 0.0, + "1665739511": 0.0, + "1747178030": 1.9092875719070435, + "1881223807": 1.9813720285892487, + "1919198188": 1.9560884833335876, + "1991414747": 0.0, + "1997090716": 0.0, + "2015956957": 0.09564426057040691, + "2054899126": 0.0, + "211245842": 0.0, + "2141802816": 1.861636370420456, + "256770874": 1.835601955652237, + "278008524": 1.1011469289660454, + "416183106": 1.812951534986496, + "534717141": 0.0, + "551504635": 1.273692786693573, + "565369392": 1.339533941199382, + "565621826": 0.0, + "585904130": 0.1701989375675718, + "626486813": 0.5741560639580712, + "636680137": 1.9821314811706543, + "706479600": 0.0, + "750448296": 0.1865733889862895, + "75070641": 1.805951327085495, + "767747424": 0.10090384539216757, + "77147264": 0.0, + "794181626": 0.0, + "957101489": 1.432861328125, + "998761162": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.5834752595424653, + "low": 0.6586342200808432, + "mean": 1.1514351414010187, + "n": 5, + "num_samples": 500 + }, + "mean": 1.1514351414010187, + "per_task": { + "LiftPegUpright-v1": 1.4578267857432365, + "PickCube-v1": 1.8952929228544235, + "PullCube-v1": 0.44455755099839095, + "PushCube-v1": 0.748361566892037, + "StackCube-v1": 1.211136880517006 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9684723974298348, + "low": 0.5435695670675014, + "mean": 0.7775791876610247, + "n": 69, + "num_samples": 500 + }, + "n": 69 + } + }, + "num_rows": 69, + "report_type": "ctt_generated_measured_rollout", + "schema_version": 1, + "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957", + "source_split_hash": "867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372", + "success_summary": { + "base_success_rate": 0.2898550724637681, + "base_utility_mean": 0.7456761826086196, + "hidden_chart_oracle_success_rate": 0.6666666666666666, + "hidden_chart_oracle_utility_mean": 1.3999846732054932, + "max_restore_error": 1.4901161193847656e-08, + "proposal_oracle_success_gain_over_base": 0.13043478260869565, + "proposal_oracle_success_rate": 0.42028985507246375, + "proposal_oracle_utility_mean": 0.9253245886521535, + "selected_success_gain_over_base": -0.028985507246376812, + "selected_success_rate": 0.2608695652173913, + "selected_utility_mean": 0.6224054855444685, + "success_selector_gap": 0.15942028985507245, + "success_support_gap": 0.2463768115942029 + }, + "target_content_hash": "fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42", + "target_split_hash": "9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381" +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_seed.json b/workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..b4aa45a2a5a7e8cd08f81809b18f2dd25b136903 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_seed.json @@ -0,0 +1,1083 @@ +{ + "1025740212": { + "base_success": 0.0, + "base_utility": 0.528373658657074, + "branch_car_at_8": 1.772506669163704, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.17388092747958847, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22749333083629608, + "selected_utility_gain_over_base_at_8": -0.3008803278207779, + "selector_regret_at_8": 1.772506669163704, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.772506669163704 + }, + "1055157354": { + "base_success": 0.0, + "base_utility": 0.19648407399654388, + "branch_car_at_8": 0.03480546176433563, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1805709977888097, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22467364370822906, + "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18986818194389343, + "selected_utility_gain_over_base_at_8": -0.006615892052650452, + "selector_regret_at_8": 0.03480546176433563, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.775326356291771, + "total_car_to_hidden_at_8": 1.8101318180561066 + }, + "1055567618": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9218642711639404, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19289744165908876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07813572883605957, + "selected_utility_gain_over_base_at_8": -1.9218642711639404, + "selector_regret_at_8": 1.9218642711639404, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9218642711639404 + }, + "1113268940": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, + "branch_car_at_8": 0.06608206033706665, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.032532089835823584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7025249004364014, + "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6364428400993347, + "selected_utility_gain_over_base_at_8": 0.21413031220436096, + "selector_regret_at_8": 0.06608206033706665, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2974750995635986, + "total_car_to_hidden_at_8": 1.3635571599006653 + }, + "1113371261": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.233029795067053, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1176943004": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.17468938592004596, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1189392917": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1231780997": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, + "branch_car_at_8": 0.5006794333457947, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10207358838016511, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7278411984443665, + "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22716176509857178, + "selected_utility_gain_over_base_at_8": -0.49984437227249146, + "selector_regret_at_8": 0.5006794333457947, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2721588015556335, + "total_car_to_hidden_at_8": 1.7728382349014282 + }, + "1271718007": { + "base_success": 0.5, + "base_utility": 1.415381133556366, + "branch_car_at_8": 0.19734752923250198, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2350719661460603, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.2635672688484192, + "proposal_oracle_utility_gain_over_base_at_8": -0.15181386470794678, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.0662197396159172, + "selected_utility_gain_over_base_at_8": -0.34916139394044876, + "selector_regret_at_8": 0.19734752923250198, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.7364327311515808, + "total_car_to_hidden_at_8": 0.9337802603840828 + }, + "1273247072": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.35906313359737396, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, + "selector_regret_at_8": 0.35906313359737396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5362297892570496, + "total_car_to_hidden_at_8": 1.8952929228544235 + }, + "1358421325": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, + "branch_car_at_8": 1.90096914768219, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.32948159974174274, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09903085231781006, + "selected_utility_gain_over_base_at_8": -0.7336714267730713, + "selector_regret_at_8": 1.90096914768219, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.90096914768219 + }, + "1375492786": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0002880295505747199, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.008822272371479922, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0008872548933140934, + "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0005992253427393734, + "selected_utility_gain_over_base_at_8": -1.9994007746572606, + "selector_regret_at_8": 0.0002880295505747199, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24256371456431225, + "total_car_to_hidden_at_8": 0.24285174411488697 + }, + "141984615": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, + "branch_car_at_8": 0.013830550014972687, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.19434281311769708, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.054234166940053306, + "proposal_oracle_utility_gain_over_base_at_8": -0.039577972143888474, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04040361692508062, + "selected_utility_gain_over_base_at_8": -0.05340852215886116, + "selector_regret_at_8": 0.013830550014972687, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.14621367553869882, + "total_car_to_hidden_at_8": 0.1600442255536715 + }, + "1475440973": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.28389310836792, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.22214884767353088, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7161068916320801, + "selected_utility_gain_over_base_at_8": -1.28389310836792, + "selector_regret_at_8": 1.28389310836792, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.28389310836792 + }, + "1483645336": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9686058908700943, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.5995435922806452, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0313941091299057, + "selected_utility_gain_over_base_at_8": -1.9686058908700943, + "selector_regret_at_8": 1.9686058908700943, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9686058908700943 + }, + "1522272686": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, + "branch_car_at_8": 0.017972871661186218, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.3276128524355476, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1949000507593155, + "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.17692717909812927, + "selected_utility_gain_over_base_at_8": -0.01783391833305359, + "selector_regret_at_8": 0.017972871661186218, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8050999492406845, + "total_car_to_hidden_at_8": 1.8230728209018707 + }, + "1528740417": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.280472755432129, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.16399362968800954, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7195272445678711, + "selected_utility_gain_over_base_at_8": -1.280472755432129, + "selector_regret_at_8": 1.280472755432129, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.280472755432129 + }, + "1549324805": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, + "branch_car_at_8": 0.01237945475926002, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.23864987006465407, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6833462389186025, + "proposal_oracle_utility_gain_over_base_at_8": -0.6954012094065547, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6709667841593424, + "selected_utility_gain_over_base_at_8": -0.7077806641658148, + "selector_regret_at_8": 0.01237945475926002, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7194042829796672, + "total_car_to_hidden_at_8": 0.7317837377389272 + }, + "1607773860": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.24177237819615016, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1665739511": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2508232858898205, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1747178030": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, + "branch_car_at_8": 0.034092940390110016, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.08595818214364334, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12480536848306656, + "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09071242809295654, + "selected_utility_gain_over_base_at_8": -0.05559416115283966, + "selector_regret_at_8": 0.034092940390110016, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8751946315169334, + "total_car_to_hidden_at_8": 1.9092875719070435 + }, + "1881223807": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.024854734539985657, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2146835718150399, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.043482705950737, + "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018627971410751343, + "selected_utility_gain_over_base_at_8": -0.013399019837379456, + "selector_regret_at_8": 0.024854734539985657, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.956517294049263, + "total_car_to_hidden_at_8": 1.9813720285892487 + }, + "1919198188": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, + "branch_car_at_8": 0.018839523196220398, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1669540555093941, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06275103986263275, + "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043911516666412354, + "selected_utility_gain_over_base_at_8": 0.0067126452922821045, + "selector_regret_at_8": 0.018839523196220398, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9372489601373672, + "total_car_to_hidden_at_8": 1.9560884833335876 + }, + "1991414747": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.11896663871153935, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1997090716": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.14617959762414967, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2015956957": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, + "branch_car_at_8": 0.005573729798197746, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.1431252893441326, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.43981083631515505, + "proposal_oracle_utility_gain_over_base_at_8": 0.3726910464465618, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.4342371065169573, + "selected_utility_gain_over_base_at_8": 0.36711731664836406, + "selector_regret_at_8": 0.005573729798197746, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.09007053077220917, + "total_car_to_hidden_at_8": 0.09564426057040691 + }, + "2054899126": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "211245842": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.13508468824583414, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1611655950546265, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2141802816": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 1.861636370420456, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.09903289408743085, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13836362957954407, + "selected_utility_gain_over_base_at_8": 0.06755638122558594, + "selector_regret_at_8": 1.861636370420456, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.861636370420456 + }, + "256770874": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, + "branch_car_at_8": 0.046216949820518494, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.046630940401554466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.21061499416828156, + "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16439804434776306, + "selected_utility_gain_over_base_at_8": -0.6913500130176544, + "selector_regret_at_8": 0.046216949820518494, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7893850058317184, + "total_car_to_hidden_at_8": 1.835601955652237 + }, + "278008524": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, + "branch_car_at_8": 0.020201382460072637, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.15535141304867436, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032865648390725255, + "proposal_oracle_utility_gain_over_base_at_8": -0.0746062200050801, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012664265930652618, + "selected_utility_gain_over_base_at_8": -0.09480760246515274, + "selector_regret_at_8": 0.020201382460072637, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.0809455465059727, + "total_car_to_hidden_at_8": 1.1011469289660454 + }, + "416183106": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, + "branch_car_at_8": 1.812951534986496, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.18072825209943494, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18704846501350403, + "selected_utility_gain_over_base_at_8": -0.013277947902679443, + "selector_regret_at_8": 1.812951534986496, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.812951534986496 + }, + "534717141": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2870871533378466, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "551504635": { + "base_success": 0.0, + "base_utility": 0.7238867878913879, + "branch_car_at_8": 1.273692786693573, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2702494805053396, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.726307213306427, + "selected_utility_gain_over_base_at_8": 0.0024204254150390625, + "selector_regret_at_8": 1.273692786693573, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.273692786693573 + }, + "565369392": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, + "branch_car_at_8": 0.04557279869914055, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.17390197648626904, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.1326704372962316, + "proposal_oracle_utility_gain_over_base_at_8": -0.6670952091614405, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.08709763859709103, + "selected_utility_gain_over_base_at_8": -0.712668007860581, + "selector_regret_at_8": 0.04557279869914055, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2939611425002415, + "total_car_to_hidden_at_8": 1.339533941199382 + }, + "565621826": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "585904130": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, + "branch_car_at_8": 0.011032561771571636, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.2625659187813512, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07848061621189117, + "proposal_oracle_utility_gain_over_base_at_8": 0.0031885181864102683, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06744805444031954, + "selected_utility_gain_over_base_at_8": -0.007844043585161367, + "selector_regret_at_8": 0.011032561771571636, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.15916637579600015, + "total_car_to_hidden_at_8": 0.1701989375675718 + }, + "626486813": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, + "branch_car_at_8": 0.031722720130346715, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.23561688988950158, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07003506645560265, + "proposal_oracle_utility_gain_over_base_at_8": -0.43540495017077774, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.03831234632525593, + "selected_utility_gain_over_base_at_8": -0.46712767030112445, + "selector_regret_at_8": 0.031722720130346715, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5432130172848701, + "total_car_to_hidden_at_8": 0.5741560639580712 + }, + "636680137": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.006491700187325478, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.12019361687075947, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.02436021901667118, + "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.017868518829345703, + "selected_utility_gain_over_base_at_8": -1.9821314811706543, + "selector_regret_at_8": 0.006491700187325478, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9756397809833288, + "total_car_to_hidden_at_8": 1.9821314811706543 + }, + "706479600": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.22468613298280815, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "750448296": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, + "branch_car_at_8": 0.01770863775163889, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.14149358436045273, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0669428389519453, + "proposal_oracle_utility_gain_over_base_at_8": -0.044590720906853676, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.049234201200306416, + "selected_utility_gain_over_base_at_8": -0.062299358658492565, + "selector_regret_at_8": 0.01770863775163889, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1688647512346506, + "total_car_to_hidden_at_8": 0.1865733889862895 + }, + "75070641": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, + "branch_car_at_8": 0.3710349500179291, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.09051828052202912, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5650836229324341, + "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.194048672914505, + "selected_utility_gain_over_base_at_8": -0.008702963590621948, + "selector_regret_at_8": 0.3710349500179291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.434916377067566, + "total_car_to_hidden_at_8": 1.805951327085495 + }, + "767747424": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, + "branch_car_at_8": 0.024658456211909652, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.14269822176062383, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11239026742987335, + "proposal_oracle_utility_gain_over_base_at_8": -0.4351339202839881, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.0877318112179637, + "selected_utility_gain_over_base_at_8": -0.45979237649589777, + "selector_regret_at_8": 0.024658456211909652, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07677002181299031, + "total_car_to_hidden_at_8": 0.10090384539216757 + }, + "77147264": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.12309184879211332, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5784205198287964, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "794181626": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "957101489": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, + "branch_car_at_8": 1.432861328125, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.21219243624964876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.567138671875, + "selected_utility_gain_over_base_at_8": 0.3631737232208252, + "selector_regret_at_8": 1.432861328125, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.432861328125 + }, + "998761162": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, + "branch_car_at_8": 1.939187914133072, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.11706618974004251, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0608120858669281, + "selected_utility_gain_over_base_at_8": -0.007004484534263611, + "selector_regret_at_8": 1.939187914133072, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.939187914133072 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_task.json b/workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..822efc76f60702db9b35cfb1341b2d3c3e7cec0f --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.066929851969083, + "branch_car_at_8": 1.029554468890031, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.16081309392707618, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.3333333333333333, + "proposal_oracle_utility_at_8": 1.5717276831467946, + "proposal_oracle_utility_gain_over_base_at_8": 0.5047978311777115, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.5421732142567635, + "selected_utility_gain_over_base_at_8": -0.5247566377123197, + "selector_regret_at_8": 1.029554468890031, + "success_selector_gap_at_8": 0.6666666666666666, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.4282723168532054, + "total_car_to_hidden_at_8": 1.4578267857432365 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.35906313359737396, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, + "selector_regret_at_8": 0.35906313359737396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5362297892570496, + "total_car_to_hidden_at_8": 1.8952929228544235 + }, + "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, + "branch_car_at_8": 0.017109284352045506, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, + "outcome_ptr_at_8": 0.35294117647058826, + "pairwise_causal_calibration_ece": 0.1721079818939321, + "proposal_oracle_success_at_8": 0.14705882352941177, + "proposal_oracle_success_gain_over_base_at_8": -0.14705882352941177, + "proposal_oracle_utility_at_8": 0.3548735701530377, + "proposal_oracle_utility_gain_over_base_at_8": -0.29635635063031635, + "selected_success_at_8": 0.14705882352941177, + "selected_success_gain_over_base_at_8": -0.14705882352941177, + "selected_utility_at_8": 0.3377642858009922, + "selected_utility_gain_over_base_at_8": -0.31346563498236185, + "selector_regret_at_8": 0.017109284352045506, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.17647058823529413, + "success_total_car_to_hidden_at_8": 0.17647058823529413, + "support_gap_at_8": 0.4276017144216252, + "total_car_to_hidden_at_8": 0.44455755099839095 + }, + "PushCube-v1": { + "base_success": 0.38461538461538464, + "base_utility": 0.8217729788560134, + "branch_car_at_8": 0.4485462617415648, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6153846153846154, + "pairwise_causal_calibration_ece": 0.15912329792465574, + "proposal_oracle_success_at_8": 0.8461538461538461, + "proposal_oracle_success_gain_over_base_at_8": 0.46153846153846156, + "proposal_oracle_utility_at_8": 1.7001846948495278, + "proposal_oracle_utility_gain_over_base_at_8": 0.8784117159935144, + "selected_success_at_8": 0.6153846153846154, + "selected_success_gain_over_base_at_8": 0.23076923076923078, + "selected_utility_at_8": 1.251638433107963, + "selected_utility_gain_over_base_at_8": 0.4298654542519496, + "selector_regret_at_8": 0.4485462617415648, + "success_selector_gap_at_8": 0.23076923076923078, + "success_support_gap_at_8": 0.15384615384615385, + "success_total_car_to_hidden_at_8": 0.38461538461538464, + "support_gap_at_8": 0.2998153051504722, + "total_car_to_hidden_at_8": 0.748361566892037 + }, + "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8047854532798131, + "branch_car_at_8": 0.5301474064588547, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.20081454850953362, + "proposal_oracle_success_at_8": 0.6, + "proposal_oracle_success_gain_over_base_at_8": 0.4, + "proposal_oracle_utility_at_8": 1.3190105259418488, + "proposal_oracle_utility_gain_over_base_at_8": 0.5142250726620357, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.13333333333333333, + "selected_utility_at_8": 0.788863119482994, + "selected_utility_gain_over_base_at_8": -0.015922333796819052, + "selector_regret_at_8": 0.5301474064588547, + "success_selector_gap_at_8": 0.26666666666666666, + "success_support_gap_at_8": 0.4, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 0.6809894740581512, + "total_car_to_hidden_at_8": 1.211136880517006 + } +} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/source_data_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed2/source_data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6cc02ca1808e0515839939b88beab0acaad5cbc --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/source_data_hash.txt @@ -0,0 +1 @@ +90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/source_split_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed2/source_split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..26988e766c6751325c7f72d33b199bd133350cde --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/source_split_hash.txt @@ -0,0 +1 @@ +867a94a1c67686c082765671a7bf8d74a9d633085e1b2619c83197c60bf58372 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/split_hash.txt b/workspace/runs/ctt_residual_rollout_val69_seed2/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..8533e80f235606cef4ba0119a3ac3806d63a0983 --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/split_hash.txt @@ -0,0 +1 @@ +9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381 diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/table.tex b/workspace/runs/ctt_residual_rollout_val69_seed2/table.tex new file mode 100644 index 0000000000000000000000000000000000000000..642f00b60f88d783018f8f64fec0cbc57c6379fa --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/table.tex @@ -0,0 +1,28 @@ +% Auto-generated by scripts/eval_metrics.py +\begin{tabular}{lrrrr} +\toprule +Metric & N & Micro mean & CI low & CI high \\ +\midrule +base\_success & 69 & 0.2899 & 0.1884 & 0.4203 \\ +base\_utility & 69 & 0.7457 & 0.5696 & 0.9620 \\ +branch\_car\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\ +hidden\_chart\_oracle\_success\_at\_8 & 69 & 0.6667 & 0.5507 & 0.7826 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 69 & 1.4000 & 1.1855 & 1.6062 \\ +outcome\_ptr\_at\_8 & 69 & 0.5072 & 0.3623 & 0.6087 \\ +pairwise\_causal\_calibration\_ece & 69 & 0.1765 & 0.1491 & 0.2027 \\ +proposal\_oracle\_success\_at\_8 & 69 & 0.4203 & 0.3188 & 0.5507 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 69 & 0.1304 & -0.0145 & 0.2609 \\ +proposal\_oracle\_utility\_at\_8 & 69 & 0.9253 & 0.7270 & 1.1546 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 69 & 0.1796 & -0.0557 & 0.4002 \\ +selected\_success\_at\_8 & 69 & 0.2609 & 0.1739 & 0.3623 \\ +selected\_success\_gain\_over\_base\_at\_8 & 69 & -0.0290 & -0.1594 & 0.0870 \\ +selected\_utility\_at\_8 & 69 & 0.6224 & 0.4473 & 0.8056 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 69 & -0.1233 & -0.3535 & 0.0836 \\ +selector\_regret\_at\_8 & 69 & 0.3029 & 0.1642 & 0.4617 \\ +success\_selector\_gap\_at\_8 & 69 & 0.1594 & 0.0870 & 0.2609 \\ +success\_support\_gap\_at\_8 & 69 & 0.2464 & 0.1449 & 0.3478 \\ +success\_total\_car\_to\_hidden\_at\_8 & 69 & 0.4058 & 0.2754 & 0.5217 \\ +support\_gap\_at\_8 & 69 & 0.4747 & 0.3004 & 0.6486 \\ +total\_car\_to\_hidden\_at\_8 & 69 & 0.7776 & 0.5436 & 0.9685 \\ +\bottomrule +\end{tabular} diff --git a/workspace/runs/ctt_residual_rollout_val69_seed2/train.log b/workspace/runs/ctt_residual_rollout_val69_seed2/train.log new file mode 100644 index 0000000000000000000000000000000000000000..250e67162b0a727b5267127aa5c4fa8a4ded65ec --- /dev/null +++ b/workspace/runs/ctt_residual_rollout_val69_seed2/train.log @@ -0,0 +1 @@ +CTT checkpoint trained separately; rollout used checkpoint=runs/ctt_residual_full_seed2/model.pt diff --git a/workspace/runs/ctt_val_rollout_comparison/command.txt b/workspace/runs/ctt_val_rollout_comparison/command.txt index c07cac97c0d8f03e6fa943a32afb15d6d448de5c..f0300a15da28bf835bf603cf4a5a8499b38c6fd3 100644 --- a/workspace/runs/ctt_val_rollout_comparison/command.txt +++ b/workspace/runs/ctt_val_rollout_comparison/command.txt @@ -1 +1 @@ -python scripts/build_ctt_rollout_comparison.py --run-glob runs/ctt_residual_rollout_val69_seed* --out-dir runs/ctt_val_rollout_comparison --k 8 --bootstrap-samples 1000 +python scripts/build_ctt_rollout_comparison.py --run-glob runs/ctt_residual_rollout_val69_seed* --out-dir runs/ctt_val_rollout_comparison --k 8 --bootstrap-samples 500 diff --git a/workspace/runs/ctt_val_rollout_comparison/data_hash.txt b/workspace/runs/ctt_val_rollout_comparison/data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..a201880dd0bef2025b061d137204bb9db2f137c2 --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/data_hash.txt @@ -0,0 +1 @@ +fa0fe0e3881ee70ce91492445871614d01b3a47fc7c576659c53ff17ddee4e42 diff --git a/workspace/runs/ctt_val_rollout_comparison/eval.log b/workspace/runs/ctt_val_rollout_comparison/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..229dbdeb9a810d36df0cf5cac788af274835290b --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/eval.log @@ -0,0 +1,3 @@ +runs/ctt_residual_rollout_val69_seed0: measured_candidates.json +runs/ctt_residual_rollout_val69_seed1: measured_candidates.json +runs/ctt_residual_rollout_val69_seed2: measured_candidates.json diff --git a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics.json b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics.json index 5bb30d720b2aa82f110305a466acb1d8811648c6..f93d1d1869dd9b43200606a8f89052f3ed2ef063 100644 --- a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics.json +++ b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics.json @@ -5,2293 +5,5973 @@ "num_rows": 207, "rows": [ { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.4897426515817642, "chart_id": "ms-484ea653bbca2de3a4a2", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18322944379343714, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7187411189079285, + "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766, "seed": "1231780997/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22899846732616425, + "selected_utility_gain_over_base_at_8": -0.498007670044899, "selector_regret_at_8": 0.4897426515817642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2812588810920715, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7710015326738358 }, { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.26188503205776215, "chart_id": "ms-4b4cd12d70c0615e0930", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08963387331564551, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4761262834072113, + "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267, "seed": "1025740212/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21424125134944916, + "selected_utility_gain_over_base_at_8": -0.3141324073076248, "selector_regret_at_8": 0.26188503205776215, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.5238737165927887, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7857587486505508 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-6e268d1e2915740816a1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.09930428958964055, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1475440973/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.7239725589752197, "branch_car_at_8": 0.033253490924835205, "chart_id": "ms-8864796e31bbe93d7c5d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10932818664818449, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7250897884368896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219, "seed": "551504635/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6918362975120544, + "selected_utility_gain_over_base_at_8": -0.03213626146316528, "selector_regret_at_8": 0.033253490924835205, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2749102115631104, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.3081637024879456 }, { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.38470593094825745, "chart_id": "ms-b04b489f4fbad93b65ca", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1616344352768096, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6646121144294739, + "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012, "seed": "1113268940/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.27990618348121643, + "selected_utility_gain_over_base_at_8": -0.14240634441375732, "selector_regret_at_8": 0.38470593094825745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.3353878855705261, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7200938165187836 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c663f7e73e7e8376a05c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.015990866806051107, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1528740417/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.0, "chart_id": "ms-2458fcfffb3f96c6ca1a", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, "seed": "1273247072/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8672154396772385, - "task_id": "PickCube-v1" + "task_id": "PickCube-v1", + "total_car_to_hidden_at_8": 1.8672154396772385 }, { + "base_success": 0.0, + "base_utility": 0.050487834960222244, "branch_car_at_8": 0.09784513339400291, "chart_id": "ms-0345d8ac530f88cefd61", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0825986241056758, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1541295200586319, + "proposal_oracle_utility_gain_over_base_at_8": 0.10364168509840965, "seed": "141984615/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05628438666462898, + "selected_utility_gain_over_base_at_8": 0.005796551704406738, "selector_regret_at_8": 0.09784513339400291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.025680318474769592, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1235254518687725 }, { + "base_success": 0.0, + "base_utility": 0.07743936777114868, "branch_car_at_8": 0.003723899833858013, "chart_id": "ms-0819eb89e627595e7e1b", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.14572584629058838, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2449440706869753, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01781417988240719, + "proposal_oracle_utility_gain_over_base_at_8": -0.05962518788874149, "seed": "2015956957/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.014090280048549175, + "selected_utility_gain_over_base_at_8": -0.0633490877225995, "selector_regret_at_8": 0.003723899833858013, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1279116664081812, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1316355662420392 }, { + "base_success": 0.0, + "base_utility": 0.19400137662887573, "branch_car_at_8": 0.0671776756644249, "chart_id": "ms-0dee962a5901fb8b5bbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1760029309478162, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.18084704875946045, + "proposal_oracle_utility_gain_over_base_at_8": -0.013154327869415283, "seed": "565369392/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.11366937309503555, + "selected_utility_gain_over_base_at_8": -0.08033200353384018, "selector_regret_at_8": 0.0671776756644249, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8191529512405396, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8863306269049644 }, { + "base_success": 0.0, + "base_utility": 0.04583283141255379, "branch_car_at_8": 0.0, "chart_id": "ms-126eb8b36769f9ba4671", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1985912322998047, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13211125003864038, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.14532220363616943, + "proposal_oracle_utility_gain_over_base_at_8": 0.09948937222361565, "seed": "767747424/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14532220363616943, + "selected_utility_gain_over_base_at_8": 0.09948937222361565, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.053269028663635254, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.053269028663635254 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-16661b2fd2c85f57a66c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2212803648424329, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002854267368093133, "chart_id": "ms-20e1efbb745fd02abc95", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, "seed": "1375492786/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, "selector_regret_at_8": 0.0002854267368093133, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24180691444780678, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2420923411846161 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-2dc5138251afd26a040d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565621826/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08252689987421036, "branch_car_at_8": 0.025417465716600418, "chart_id": "ms-387e6771a55be98f1a59", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20744429528713226, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08046608951440049, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08211058378219604, + "proposal_oracle_utility_gain_over_base_at_8": -0.00041631609201431274, "seed": "767747424/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05669311806559563, + "selected_utility_gain_over_base_at_8": -0.02583378180861473, "selector_regret_at_8": 0.025417465716600418, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.12533371150493622, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15075117722153664 }, { + "base_success": 0.0, + "base_utility": 0.03917040303349495, "branch_car_at_8": 0.003450155258178711, "chart_id": "ms-38f7298184a2c5005e94", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1767096221446991, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12670819487819776, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03909116983413696, + "proposal_oracle_utility_gain_over_base_at_8": -7.923319935798645e-05, "seed": "2015956957/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03564101457595825, + "selected_utility_gain_over_base_at_8": -0.0035293884575366974, "selector_regret_at_8": 0.003450155258178711, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13761845231056213, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.14106860756874084 }, { + "base_success": 0.0, + "base_utility": 0.009124159812927246, "branch_car_at_8": 0.009075979702174664, "chart_id": "ms-482c5c8d208b5907a2c2", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1834145486354828, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.06830739992931822, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.014053802005946636, + "proposal_oracle_utility_gain_over_base_at_8": 0.00492964219301939, "seed": "626486813/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.004977822303771973, + "selected_utility_gain_over_base_at_8": -0.0041463375091552734, "selector_regret_at_8": 0.009075979702174664, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16936074662953615, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.17843672633171082 }, { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.0, "chart_id": "ms-4da03300d92420e80ed0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15825371494561297, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038, "seed": "1747178030/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8536934107542038, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-57be9ae181c077e7821f", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.142949998938739, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.1544450968503952, "branch_car_at_8": 0.0, "chart_id": "ms-6388b5d2356c556c3e55", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09390309844881675, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8455549031496048, "seed": "2015956957/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8455549031496048, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.06173701956868172, "branch_car_at_8": 0.038424551486968994, "chart_id": "ms-638d2ba1917eff4d09e0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.11176212877035141, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.47662120577716466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.054154932498931885, + "proposal_oracle_utility_gain_over_base_at_8": -0.007582087069749832, "seed": "767747424/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01573038101196289, + "selected_utility_gain_over_base_at_8": -0.046006638556718826, "selector_regret_at_8": 0.038424551486968994, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.057607196271419525, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.09603174775838852 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-652ad552ba98d2736620", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2367449700832367, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15432521715768022, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "767747424/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03784912824630737, "branch_car_at_8": 0.04759359359741211, "chart_id": "ms-674fad0e7c042d23629e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.15573063492774963, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.3308470025433796, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11711406707763672, + "proposal_oracle_utility_gain_over_base_at_8": 0.07926493883132935, "seed": "2015956957/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06952047348022461, + "selected_utility_gain_over_base_at_8": 0.031671345233917236, "selector_regret_at_8": 0.04759359359741211, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.038616567850112915, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.08621016144752502 }, { + "base_success": 0.0, + "base_utility": 0.12791956961154938, "branch_car_at_8": 0.13747439347207546, "chart_id": "ms-728b865677134d3788e3", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.22658847272396088, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15394090571485494, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1656186729669571, + "proposal_oracle_utility_gain_over_base_at_8": 0.037699103355407715, "seed": "750448296/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02814427949488163, + "selected_utility_gain_over_base_at_8": -0.09977529011666775, "selector_regret_at_8": 0.13747439347207546, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.060969799757003784, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.19844419322907925 }, { + "base_success": 0.0, + "base_utility": 0.20529556274414062, "branch_car_at_8": 0.1261332780122757, "chart_id": "ms-7ea98b8f4d2220347ee0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.27989473938941956, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.25933831347005637, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.17496724426746368, + "proposal_oracle_utility_gain_over_base_at_8": -0.03032831847667694, "seed": "565369392/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04883396625518799, + "selected_utility_gain_over_base_at_8": -0.15646159648895264, "selector_regret_at_8": 0.1261332780122757, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.10492749512195587, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.23106077313423157 }, { + "base_success": 0.0, + "base_utility": 0.18213583528995514, "branch_car_at_8": 0.0009371041087433696, "chart_id": "ms-7f57045c114ddad153b5", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.227622389793396, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12290390946733898, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0028197963256388903, + "proposal_oracle_utility_gain_over_base_at_8": -0.17931603896431625, "seed": "278008524/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0018826922168955207, + "selected_utility_gain_over_base_at_8": -0.18025314307305962, "selector_regret_at_8": 0.0009371041087433696, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2248025934677571, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22573969757650048 }, { + "base_success": 0.0, + "base_utility": 0.0056487321853637695, "branch_car_at_8": 0.004609187599271536, "chart_id": "ms-81f4e8858b2ba90aea2e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1631626933813095, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1605827188396785, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01193171739578247, + "proposal_oracle_utility_gain_over_base_at_8": 0.006282985210418701, "seed": "626486813/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007322529796510935, + "selected_utility_gain_over_base_at_8": 0.0016737976111471653, "selector_regret_at_8": 0.004609187599271536, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.15123097598552704, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15584016358479857 }, { + "base_success": 0.0, + "base_utility": 0.00698717450723052, "branch_car_at_8": 0.06059660017490387, "chart_id": "ms-8a72b73ac452bc190aeb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.10329639911651611, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.25922105796565, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22011399269104004, + "proposal_oracle_utility_gain_over_base_at_8": 0.21312681818380952, "seed": "626486813/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15951739251613617, + "selected_utility_gain_over_base_at_8": 0.15253021800890565, "selector_regret_at_8": 0.06059660017490387, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08285123109817505, "branch_car_at_8": 0.0, "chart_id": "ms-a96bca1ef0b74f686018", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1572066992521286, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1273789448476313, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08222240209579468, + "proposal_oracle_utility_gain_over_base_at_8": -0.0006288290023803711, "seed": "141984615/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08222240209579468, + "selected_utility_gain_over_base_at_8": -0.0006288290023803711, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.07498429715633392, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.07498429715633392 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-ac4fb33f3ab4db853995", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.23392411403598115, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565369392/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c31b98b3d3702c6f9c8c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "2054899126/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.037537217140197754, "branch_car_at_8": 0.0518533973954618, "chart_id": "ms-c832439976a37f52d094", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2594742774963379, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.067807253260605, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.056055743247270584, + "proposal_oracle_utility_gain_over_base_at_8": 0.01851852610707283, "seed": "585904130/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.004202345851808786, + "selected_utility_gain_over_base_at_8": -0.03333487128838897, "selector_regret_at_8": 0.0518533973954618, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2034185342490673, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2552719316445291 }, { + "base_success": 0.0, + "base_utility": 0.09138330072164536, "branch_car_at_8": 0.010477880015969276, "chart_id": "ms-c96b415f083f60857334", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.26426148414611816, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1353708462114277, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01781187392771244, + "proposal_oracle_utility_gain_over_base_at_8": -0.07357142679393291, "seed": "585904130/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007333993911743164, + "selected_utility_gain_over_base_at_8": -0.08404930680990219, "selector_regret_at_8": 0.010477880015969276, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24644961021840572, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.256927490234375 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9274959191679955, "chart_id": "ms-d73fd5fce55fb58ef8e9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.6159370955924921, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "534717141/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07250408083200455, + "selected_utility_gain_over_base_at_8": -1.9274959191679955, "selector_regret_at_8": 1.9274959191679955, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9274959191679955 }, { + "base_success": 0.0, + "base_utility": 0.14809735119342804, "branch_car_at_8": 0.0016585588455200195, "chart_id": "ms-da142025733b53fcf7eb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2643269896507263, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11781174912716066, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0028871893882751465, + "proposal_oracle_utility_gain_over_base_at_8": -0.1452101618051529, "seed": "141984615/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.001228630542755127, + "selected_utility_gain_over_base_at_8": -0.1468687206506729, "selector_regret_at_8": 0.0016585588455200195, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.26143980026245117, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2630983591079712 }, { + "base_success": 0.0, + "base_utility": 0.1362423449754715, "branch_car_at_8": 0.0102931447327137, "chart_id": "ms-e2bad6e99e8a281207cc", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20825156569480896, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.21725827933405525, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.028299549594521523, + "proposal_oracle_utility_gain_over_base_at_8": -0.10794279538094997, "seed": "1549324805/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018006404861807823, + "selected_utility_gain_over_base_at_8": -0.11823594011366367, "selector_regret_at_8": 0.0102931447327137, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.17995201610028744, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.19024516083300114 }, { + "base_success": 0.0, + "base_utility": 0.09695577621459961, "branch_car_at_8": 0.06717542745172977, "chart_id": "ms-e30fd80adc089e9f697a", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18920521438121796, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.10857618383415296, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09409594535827637, + "proposal_oracle_utility_gain_over_base_at_8": -0.002859830856323242, "seed": "585904130/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.026920517906546593, + "selected_utility_gain_over_base_at_8": -0.07003525830805302, "selector_regret_at_8": 0.06717542745172977, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09510926902294159, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.16228469647467136 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-e9ed5d488afdfd6449d1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15838532590498614, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "626486813/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.09514755010604858, "branch_car_at_8": 0.01717013120651245, "chart_id": "ms-ec013396e4aff6170ee1", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24502670764923096, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13327079696269858, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04836767911911011, + "proposal_oracle_utility_gain_over_base_at_8": -0.04677987098693848, "seed": "750448296/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.031197547912597656, + "selected_utility_gain_over_base_at_8": -0.06395000219345093, "selector_regret_at_8": 0.01717013120651245, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.19665902853012085, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2138291597366333 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-ee30ebaf98622759d888", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1699944666253151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012414653785526752, + "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732, "seed": "636680137/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.012414653785526752, + "selected_utility_gain_over_base_at_8": -1.9875853462144732, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9875853462144732, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9875853462144732 }, { + "base_success": 0.0, + "base_utility": 0.026694953441619873, "branch_car_at_8": 0.03375059738755226, "chart_id": "ms-f3f1d07cfd59b823da47", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.17124073207378387, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13684733372604946, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0821961984038353, + "proposal_oracle_utility_gain_over_base_at_8": 0.055501244962215424, "seed": "2015956957/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.048445601016283035, + "selected_utility_gain_over_base_at_8": 0.021750647574663162, "selector_regret_at_8": 0.03375059738755226, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.08904453366994858, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.12279513105750084 }, { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.07543906569480896, "chart_id": "ms-081952a63087fc8f83cd", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, "seed": "2141802816/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, "selector_regret_at_8": 0.07543906569480896, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8752704411745071, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.950709506869316 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9700170010328293, "chart_id": "ms-121bc2e6676c1ac10911", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07610219762653536, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "77147264/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.029982998967170715, + "selected_utility_gain_over_base_at_8": -1.9700170010328293, "selector_regret_at_8": 1.9700170010328293, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9700170010328293 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-22f153f0d4fa656018e6", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1271718007/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.020236089825630188, "chart_id": "ms-27c9f70e97b8850cc3ab", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, "seed": "1881223807/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, "selector_regret_at_8": 0.020236089825630188, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9486478716135025, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9688839614391327 }, { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, "chart_id": "ms-2867deaca7e98582620b", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, "seed": "1189392917/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 1.9691698104143143, "chart_id": "ms-365d8c6de6de16f66744", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, "seed": "1997090716/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, "selector_regret_at_8": 1.9691698104143143, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9691698104143143 }, { + "base_success": 0.0, + "base_utility": 0.03280790150165558, "branch_car_at_8": 0.002354145050048828, "chart_id": "ms-3c326d05fec66e1d6bea", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11560099885411955, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0200461745262146, + "proposal_oracle_utility_gain_over_base_at_8": -0.012761726975440979, "seed": "278008524/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01769202947616577, + "selected_utility_gain_over_base_at_8": -0.015115872025489807, "selector_regret_at_8": 0.002354145050048828, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9799538254737854, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9823079705238342 }, { + "base_success": 0.0, + "base_utility": 0.1733371466398239, "branch_car_at_8": 0.0, "chart_id": "ms-4dbdc6ed61dccebbfe92", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12119924562800825, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176, "seed": "794181626/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.826662853360176, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-58950fed04d203b97ab9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3179396489681421, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1113371261/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.01641605794429779, "branch_car_at_8": 0.0, "chart_id": "ms-651bc0395d78d2819158", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022, "seed": "1483645336/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.9835839420557022, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, "chart_id": "ms-9fad82fd288e4296e354", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2176350935010736, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, "seed": "1176943004/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0689050555229187, "branch_car_at_8": 0.07749588787555695, "chart_id": "ms-b8bfdd3e6c31bd98082d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.23606164078349207, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08403940498828888, + "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178, "seed": "998761162/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006543517112731934, + "selected_utility_gain_over_base_at_8": -0.06236153841018677, "selector_regret_at_8": 0.07749588787555695, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9159605950117111, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.993456482887268 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-fc8084d5cd7da937fc02", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.05287821364799471, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1991414747/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.07482179999351501, "chart_id": "ms-3afd8959363e917591d4", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2497251397544613, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2396746575832367, + "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974, "seed": "75070641/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16485285758972168, + "selected_utility_gain_over_base_at_8": -0.03789877891540527, "selector_regret_at_8": 0.07482179999351501, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7603253424167633, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8351471424102783 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.183156132698059, "chart_id": "ms-3ff1aedb72f65a182b58", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07486610854656817, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1055567618/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8168438673019409, + "selected_utility_gain_over_base_at_8": -1.183156132698059, "selector_regret_at_8": 1.183156132698059, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.183156132698059 }, { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 1.8516457676887512, "chart_id": "ms-4048f8f6e047422e3c79", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2973966626262834, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087, "seed": "1055157354/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14835423231124878, + "selected_utility_gain_over_base_at_8": -0.02813686430454254, "selector_regret_at_8": 1.8516457676887512, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8516457676887512 }, { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.009524375200271606, "chart_id": "ms-5b26c33feed6992386b1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.08257819446483214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2435554713010788, + "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935, "seed": "1522272686/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2340310961008072, + "selected_utility_gain_over_base_at_8": 0.03926999866962433, "selector_regret_at_8": 0.009524375200271606, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7564445286989212, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7659689038991928 }, { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.5908347219228745, "chart_id": "ms-63ac245579b1188935ad", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16771630653845432, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8130142092704773, + "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624, "seed": "211245842/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22217948734760284, + "selected_utility_gain_over_base_at_8": -0.6166549175977707, "selector_regret_at_8": 0.5908347219228745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1869857907295227, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7778205126523972 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.007551252841949463, "chart_id": "ms-6c8d06bace98a28a8e4e", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.050940808914706356, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8216220736503601, + "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964, "seed": "1665739511/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8140708208084106, + "selected_utility_gain_over_base_at_8": -1.1859291791915894, "selector_regret_at_8": 0.007551252841949463, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.17837792634964, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.1859291791915894 }, { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.10820980370044708, "chart_id": "ms-72777024349c2aacd5e0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.030709639081801327, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.24107472598552704, + "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904, "seed": "256770874/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13286492228507996, + "selected_utility_gain_over_base_at_8": -0.7228831350803375, "selector_regret_at_8": 0.10820980370044708, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.758925274014473, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.86713507771492 }, { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 0.0, "chart_id": "ms-7fc5696cb3dc7b9bb9f7", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.20250855048512742, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5881698727607727, + "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979, "seed": "957101489/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5881698727607727, + "selected_utility_gain_over_base_at_8": 0.3842049241065979, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.4118301272392273, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.4118301272392273 }, { + "base_success": 0.0, + "base_utility": 0.8431589603424072, "branch_car_at_8": 1.7671720832586288, "chart_id": "ms-9a14552b7db695dee92c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.05415195874099398, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1568410396575928, "seed": "77147264/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23282791674137115, + "selected_utility_gain_over_base_at_8": -0.6103310436010361, "selector_regret_at_8": 1.7671720832586288, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7671720832586288 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.779732346534729, "chart_id": "ms-9f5d57fe2153fd267774", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08588083120555855, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1607773860/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.220267653465271, + "selected_utility_gain_over_base_at_8": -1.779732346534729, "selector_regret_at_8": 1.779732346534729, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.779732346534729 }, { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 0.5775325745344162, "chart_id": "ms-b0fee26e1db967f90b81", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13502516765102715, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8142459392547607, + "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605, "seed": "1358421325/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23671336472034454, + "selected_utility_gain_over_base_at_8": -0.5959889143705368, "selector_regret_at_8": 0.5775325745344162, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1857540607452393, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7632866352796555 }, { + "base_success": 0.0, + "base_utility": 0.8285209536552429, "branch_car_at_8": 0.0, "chart_id": "ms-bc2d3b425d9163a62dbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1508652338231993, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.171479046344757, "seed": "1271718007/train0", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.171479046344757, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 0.3360496461391449, "chart_id": "ms-d1960368ef76bbb10669", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10129212810935397, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5778142213821411, + "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764, "seed": "416183106/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.24176457524299622, + "selected_utility_gain_over_base_at_8": 0.041438162326812744, "selector_regret_at_8": 0.3360496461391449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.4221857786178589, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7582354247570038 }, { + "base_success": 0.0, + "base_utility": 0.8125406503677368, "branch_car_at_8": 1.1865743398666382, "chart_id": "ms-efdc07c985935d871adf", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0922387871049479, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632, "seed": "706479600/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8134256601333618, + "selected_utility_gain_over_base_at_8": 0.000885009765625, "selector_regret_at_8": 1.1865743398666382, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.1865743398666382 }, { + "base_success": 0.0, + "base_utility": 0.037198856472969055, "branch_car_at_8": 0.016854479908943176, "chart_id": "ms-fd2387aeed9bfc9a0d08", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15779226512372985, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08992795646190643, + "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738, "seed": "1919198188/train0", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07307347655296326, + "selected_utility_gain_over_base_at_8": 0.0358746200799942, "selector_regret_at_8": 0.016854479908943176, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9100720435380936, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9269265234470367 }, { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.0, "chart_id": "ms-484ea653bbca2de3a4a2", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28322114525972475, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368, "seed": "1231780997/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.2729938626289368, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.47739143669605255, "chart_id": "ms-4b4cd12d70c0615e0930", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12992986623548633, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6983401775360107, + "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677, "seed": "1025740212/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2209487408399582, + "selected_utility_gain_over_base_at_8": -0.3074249178171158, "selector_regret_at_8": 0.47739143669605255, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.3016598224639893, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7790512591600418 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.289921224117279, "chart_id": "ms-6e268d1e2915740816a1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2443552809335613, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1475440973/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.710078775882721, + "selected_utility_gain_over_base_at_8": -1.289921224117279, "selector_regret_at_8": 1.289921224117279, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.289921224117279 }, { + "base_success": 0.0, + "base_utility": 0.7247228622436523, "branch_car_at_8": 0.0, "chart_id": "ms-8864796e31bbe93d7c5d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2299325778712667, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7224790453910828, + "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958, "seed": "551504635/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7224790453910828, + "selected_utility_gain_over_base_at_8": -0.00224381685256958, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2775209546089172, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.2775209546089172 }, { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.18306860327720642, "chart_id": "ms-b04b489f4fbad93b65ca", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10725283874677279, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6617497801780701, + "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963, "seed": "1113268940/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.47868117690086365, + "selected_utility_gain_over_base_at_8": 0.05636864900588989, "selector_regret_at_8": 0.18306860327720642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.33825021982193, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.5213188230991364 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c663f7e73e7e8376a05c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2757671830981365, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1528740417/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.45726464316248894, "chart_id": "ms-2458fcfffb3f96c6ca1a", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, "seed": "1273247072/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, "selector_regret_at_8": 0.45726464316248894, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.500250667333603, - "task_id": "PickCube-v1" + "task_id": "PickCube-v1", + "total_car_to_hidden_at_8": 1.9575153104960918 }, { + "base_success": 0.0, + "base_utility": 0.050487834960222244, "branch_car_at_8": 0.13774145022034645, "chart_id": "ms-0345d8ac530f88cefd61", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10996723271348506, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.18969689309597015, + "proposal_oracle_utility_gain_over_base_at_8": 0.1392090581357479, "seed": "141984615/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0519554428756237, + "selected_utility_gain_over_base_at_8": 0.0014676079154014587, "selector_regret_at_8": 0.13774145022034645, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1278543956577778 }, { + "base_success": 0.0, + "base_utility": 0.07743936777114868, "branch_car_at_8": 0.004675229080021381, "chart_id": "ms-0819eb89e627595e7e1b", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.14572584629058838, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11537806751098544, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.012503445148468018, + "proposal_oracle_utility_gain_over_base_at_8": -0.06493592262268066, "seed": "2015956957/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007828216068446636, + "selected_utility_gain_over_base_at_8": -0.06961115170270205, "selector_regret_at_8": 0.004675229080021381, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13322240114212036, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13789763022214174 }, { + "base_success": 0.0, + "base_utility": 0.19400137662887573, "branch_car_at_8": 0.04213055223226547, "chart_id": "ms-0dee962a5901fb8b5bbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.025648459676823474, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.16528861224651337, + "proposal_oracle_utility_gain_over_base_at_8": -0.028712764382362366, "seed": "565369392/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1231580600142479, + "selected_utility_gain_over_base_at_8": -0.07084331661462784, "selector_regret_at_8": 0.04213055223226547, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8347113877534866, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.876841939985752 }, { + "base_success": 0.0, + "base_utility": 0.04583283141255379, "branch_car_at_8": 0.06697340309619904, "chart_id": "ms-126eb8b36769f9ba4671", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1985912322998047, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.04387270095990371, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.14882762730121613, + "proposal_oracle_utility_gain_over_base_at_8": 0.10299479588866234, "seed": "767747424/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08185422420501709, + "selected_utility_gain_over_base_at_8": 0.0360213927924633, "selector_regret_at_8": 0.06697340309619904, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.04976360499858856, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1167370080947876 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-16661b2fd2c85f57a66c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.5994816934153381, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.00047226750757545233, "chart_id": "ms-20e1efbb745fd02abc95", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14027669602920534, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0017900466918945312, + "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055, "seed": "1375492786/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001317779184319079, + "selected_utility_gain_over_base_at_8": -1.998682220815681, "selector_regret_at_8": 0.00047226750757545233, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2416609227657318, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24213319027330726 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-2dc5138251afd26a040d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565621826/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08252689987421036, "branch_car_at_8": 0.07240533083677292, "chart_id": "ms-387e6771a55be98f1a59", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20744429528713226, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11885014434276354, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.14194543659687042, + "proposal_oracle_utility_gain_over_base_at_8": 0.059418536722660065, "seed": "767747424/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0695401057600975, + "selected_utility_gain_over_base_at_8": -0.012986794114112854, "selector_regret_at_8": 0.07240533083677292, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.06549885869026184, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13790418952703476 }, { + "base_success": 0.0, + "base_utility": 0.03917040303349495, "branch_car_at_8": 0.011704225093126297, "chart_id": "ms-38f7298184a2c5005e94", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1767096221446991, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2012286798976311, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.052575647830963135, + "proposal_oracle_utility_gain_over_base_at_8": 0.013405244797468185, "seed": "2015956957/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04087142273783684, + "selected_utility_gain_over_base_at_8": 0.0017010197043418884, "selector_regret_at_8": 0.011704225093126297, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.12413397431373596, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13583819940686226 }, { + "base_success": 0.0, + "base_utility": 0.009124159812927246, "branch_car_at_8": 0.002153436653316021, "chart_id": "ms-482c5c8d208b5907a2c2", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1834145486354828, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1426465578375924, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.008332650177180767, + "proposal_oracle_utility_gain_over_base_at_8": -0.000791509635746479, "seed": "626486813/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006179213523864746, + "selected_utility_gain_over_base_at_8": -0.0029449462890625, "selector_regret_at_8": 0.002153436653316021, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.17508189845830202, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.17723533511161804 }, { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.027157030999660492, "chart_id": "ms-4da03300d92420e80ed0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3089551740932353, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11134227365255356, + "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645, "seed": "1747178030/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08418524265289307, + "selected_utility_gain_over_base_at_8": -0.06212134659290314, "selector_regret_at_8": 0.027157030999660492, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8886577263474464, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.915814757347107 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.00473870849236846, "chart_id": "ms-57be9ae181c077e7821f", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12392755865705843, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.011638104915618896, + "proposal_oracle_utility_gain_over_base_at_8": -1.988361895084381, "seed": "1549324805/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.006899396423250437, + "selected_utility_gain_over_base_at_8": -1.9931006035767496, "selector_regret_at_8": 0.00473870849236846, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.988361895084381, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9931006035767496 }, { + "base_success": 0.0, + "base_utility": 0.1544450968503952, "branch_car_at_8": 0.006060939282178879, "chart_id": "ms-6388b5d2356c556c3e55", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.018031511869928168, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04087907075881958, + "proposal_oracle_utility_gain_over_base_at_8": -0.11356602609157562, "seed": "2015956957/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0348181314766407, + "selected_utility_gain_over_base_at_8": -0.1196269653737545, "selector_regret_at_8": 0.006060939282178879, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9591209292411804, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9651818685233593 }, { + "base_success": 0.0, + "base_utility": 0.06173701956868172, "branch_car_at_8": 0.043069109320640564, "chart_id": "ms-638d2ba1917eff4d09e0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.11176212877035141, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09922946147365272, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1519772857427597, + "proposal_oracle_utility_gain_over_base_at_8": 0.09024026617407799, "seed": "767747424/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10890817642211914, + "selected_utility_gain_over_base_at_8": 0.047171156853437424, "selector_regret_at_8": 0.043069109320640564, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0028539523482322693 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-652ad552ba98d2736620", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2367449700832367, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.03930353879708036, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.010964850895106792, + "proposal_oracle_utility_gain_over_base_at_8": -1.9890351491048932, "seed": "767747424/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.010964850895106792, + "selected_utility_gain_over_base_at_8": -1.9890351491048932, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2257801191881299, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2257801191881299 }, { + "base_success": 0.0, + "base_utility": 0.03784912824630737, "branch_car_at_8": 0.014707427471876144, "chart_id": "ms-674fad0e7c042d23629e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.15573063492774963, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.06740395181889638, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05826536938548088, + "proposal_oracle_utility_gain_over_base_at_8": 0.020416241139173508, "seed": "2015956957/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043557941913604736, + "selected_utility_gain_over_base_at_8": 0.005708813667297363, "selector_regret_at_8": 0.014707427471876144, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09746526554226875, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1121726930141449 }, { + "base_success": 0.0, + "base_utility": 0.12791956961154938, "branch_car_at_8": 0.041699863970279694, "chart_id": "ms-728b865677134d3788e3", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.22658847272396088, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.36236752551402057, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06523416191339493, + "proposal_oracle_utility_gain_over_base_at_8": -0.06268540769815445, "seed": "750448296/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.023534297943115234, + "selected_utility_gain_over_base_at_8": -0.10438527166843414, "selector_regret_at_8": 0.041699863970279694, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16135431081056595, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.20305417478084564 }, { + "base_success": 0.0, + "base_utility": 0.20529556274414062, "branch_car_at_8": 0.03670026361942291, "chart_id": "ms-7ea98b8f4d2220347ee0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.27989473938941956, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.34662446466496566, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1852296143770218, + "proposal_oracle_utility_gain_over_base_at_8": -0.020065948367118835, "seed": "565369392/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14852935075759888, + "selected_utility_gain_over_base_at_8": -0.05676621198654175, "selector_regret_at_8": 0.03670026361942291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09466512501239777, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.13136538863182068 }, { + "base_success": 0.0, + "base_utility": 0.18213583528995514, "branch_car_at_8": 0.0040488241938874125, "chart_id": "ms-7f57045c114ddad153b5", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.227622389793396, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08721578826990975, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.005460778716951609, + "proposal_oracle_utility_gain_over_base_at_8": -0.17667505657300353, "seed": "278008524/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0014119545230641961, + "selected_utility_gain_over_base_at_8": -0.18072388076689094, "selector_regret_at_8": 0.0040488241938874125, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2221616110764444, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2262104352703318 }, { + "base_success": 0.0, + "base_utility": 0.0056487321853637695, "branch_car_at_8": 0.010425567161291838, "chart_id": "ms-81f4e8858b2ba90aea2e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1631626933813095, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.19758543362123893, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.014848490245640278, + "proposal_oracle_utility_gain_over_base_at_8": 0.009199758060276508, "seed": "626486813/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.00442292308434844, + "selected_utility_gain_over_base_at_8": -0.0012258091010153294, "selector_regret_at_8": 0.010425567161291838, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.14831420313566923, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15873977029696107 }, { + "base_success": 0.0, + "base_utility": 0.00698717450723052, "branch_car_at_8": 0.07316120713949203, "chart_id": "ms-8a72b73ac452bc190aeb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.10329639911651611, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.44087366875059886, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11383946985006332, + "proposal_oracle_utility_gain_over_base_at_8": 0.1068522953428328, "seed": "626486813/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04067826271057129, + "selected_utility_gain_over_base_at_8": 0.03369108820334077, "selector_regret_at_8": 0.07316120713949203, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.06261813640594482 }, { + "base_success": 0.0, + "base_utility": 0.08285123109817505, "branch_car_at_8": 0.034163929522037506, "chart_id": "ms-a96bca1ef0b74f686018", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1572066992521286, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0563221293249205, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08467630296945572, + "proposal_oracle_utility_gain_over_base_at_8": 0.0018250718712806702, "seed": "141984615/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.05051237344741821, + "selected_utility_gain_over_base_at_8": -0.032338857650756836, "selector_regret_at_8": 0.034163929522037506, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.07253039628267288, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.10669432580471039 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.05995168536901474, "chart_id": "ms-ac4fb33f3ab4db853995", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0474766148185784, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.17047619819641113, + "proposal_oracle_utility_gain_over_base_at_8": -1.8295238018035889, "seed": "565369392/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.11052451282739639, + "selected_utility_gain_over_base_at_8": -1.8894754871726036, "selector_regret_at_8": 0.05995168536901474, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8295238018035889, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8894754871726036 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c31b98b3d3702c6f9c8c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "2054899126/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.037537217140197754, "branch_car_at_8": 0.0029224762693047523, "chart_id": "ms-c832439976a37f52d094", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2594742774963379, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.011460382589838352, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.01820494793355465, + "proposal_oracle_utility_gain_over_base_at_8": -0.019332269206643105, "seed": "585904130/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.015282471664249897, + "selected_utility_gain_over_base_at_8": -0.022254745475947857, "selector_regret_at_8": 0.0029224762693047523, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24126932956278324, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.244191805832088 }, { + "base_success": 0.0, + "base_utility": 0.09138330072164536, "branch_car_at_8": 0.008578261360526085, "chart_id": "ms-c96b415f083f60857334", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.26426148414611816, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.06282513857710854, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.024076303467154503, + "proposal_oracle_utility_gain_over_base_at_8": -0.06730699725449085, "seed": "585904130/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.015498042106628418, + "selected_utility_gain_over_base_at_8": -0.07588525861501694, "selector_regret_at_8": 0.008578261360526085, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24018518067896366, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24876344203948975 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9558932781219482, "chart_id": "ms-d73fd5fce55fb58ef8e9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14369576234023818, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "534717141/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.04410672187805176, + "selected_utility_gain_over_base_at_8": -1.9558932781219482, "selector_regret_at_8": 1.9558932781219482, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9558932781219482 }, { + "base_success": 0.0, + "base_utility": 0.14809735119342804, "branch_car_at_8": 0.0, "chart_id": "ms-da142025733b53fcf7eb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2643269896507263, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08970759701490164, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0028313398361206055, + "proposal_oracle_utility_gain_over_base_at_8": -0.14526601135730743, "seed": "141984615/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0028313398361206055, + "selected_utility_gain_over_base_at_8": -0.14526601135730743, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2614956498146057, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2614956498146057 }, { + "base_success": 0.0, + "base_utility": 0.1362423449754715, "branch_car_at_8": 0.0, "chart_id": "ms-e2bad6e99e8a281207cc", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20825156569480896, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13534544179994237, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.023882150650024414, + "proposal_oracle_utility_gain_over_base_at_8": -0.11236019432544708, "seed": "1549324805/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.023882150650024414, + "selected_utility_gain_over_base_at_8": -0.11236019432544708, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.18436941504478455, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.18436941504478455 }, { + "base_success": 0.0, + "base_utility": 0.09695577621459961, "branch_car_at_8": 0.024704236537218094, "chart_id": "ms-e30fd80adc089e9f697a", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18920521438121796, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14400407985489305, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0564744658768177, + "proposal_oracle_utility_gain_over_base_at_8": -0.040481310337781906, "seed": "585904130/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03177022933959961, + "selected_utility_gain_over_base_at_8": -0.065185546875, "selector_regret_at_8": 0.024704236537218094, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13273074850440025, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15743498504161835 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.014290884137153625, "chart_id": "ms-e9ed5d488afdfd6449d1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18713029784839189, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.15760482847690582, + "proposal_oracle_utility_gain_over_base_at_8": -1.8423951715230942, "seed": "626486813/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.1433139443397522, + "selected_utility_gain_over_base_at_8": -1.8566860556602478, "selector_regret_at_8": 0.014290884137153625, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8423951715230942, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8566860556602478 }, { + "base_success": 0.0, + "base_utility": 0.09514755010604858, "branch_car_at_8": 0.012320853769779205, "chart_id": "ms-ec013396e4aff6170ee1", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24502670764923096, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.10651513414126283, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08161912113428116, + "proposal_oracle_utility_gain_over_base_at_8": -0.013528428971767426, "seed": "750448296/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06929826736450195, + "selected_utility_gain_over_base_at_8": -0.02584928274154663, "selector_regret_at_8": 0.012320853769779205, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1634075865149498, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.175728440284729 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-ee30ebaf98622759d888", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07628525991805817, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.013928472995758057, + "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242, "seed": "636680137/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013928472995758057, + "selected_utility_gain_over_base_at_8": -1.986071527004242, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.986071527004242, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.986071527004242 }, { + "base_success": 0.0, + "base_utility": 0.026694953441619873, "branch_car_at_8": 0.073831457644701, "chart_id": "ms-f3f1d07cfd59b823da47", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.17124073207378387, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.41381875369242904, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11723694950342178, + "proposal_oracle_utility_gain_over_base_at_8": 0.09054199606180191, "seed": "2015956957/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04340549185872078, + "selected_utility_gain_over_base_at_8": 0.016710538417100906, "selector_regret_at_8": 0.073831457644701, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.05400378257036209, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.1278352402150631 }, { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.019766509532928467, "chart_id": "ms-081952a63087fc8f83cd", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07786951284417709, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09589323401451111, + "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298, "seed": "2141802816/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07612672448158264, + "selected_utility_gain_over_base_at_8": 0.005319476127624512, "selector_regret_at_8": 0.019766509532928467, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.904106765985489, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9238732755184174 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-121bc2e6676c1ac10911", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.389114397227515, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "77147264/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-22f153f0d4fa656018e6", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1271718007/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.009028241038322449, "chart_id": "ms-27c9f70e97b8850cc3ab", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13674585276395504, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.034886181354522705, + "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067, "seed": "1881223807/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.025857940316200256, + "selected_utility_gain_over_base_at_8": -0.006169050931930542, "selector_regret_at_8": 0.009028241038322449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9651138186454773, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9741420596837997 }, { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, "chart_id": "ms-2867deaca7e98582620b", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2180235973132697, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, "seed": "1189392917/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, "chart_id": "ms-365d8c6de6de16f66744", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2554442550173647, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, "seed": "1997090716/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03280790150165558, "branch_car_at_8": 0.01877032220363617, "chart_id": "ms-3c326d05fec66e1d6bea", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.40489593467151774, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05748283863067627, + "proposal_oracle_utility_gain_over_base_at_8": 0.02467493712902069, "seed": "278008524/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0387125164270401, + "selected_utility_gain_over_base_at_8": 0.0059046149253845215, "selector_regret_at_8": 0.01877032220363617, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9425171613693237, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.96128748357296 }, { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, "chart_id": "ms-4dbdc6ed61dccebbfe92", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28339362113630795, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, "seed": "794181626/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-58950fed04d203b97ab9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.04977883974023356, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1113371261/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9307887554168701, "chart_id": "ms-651bc0395d78d2819158", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13752192684190806, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1483645336/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.06921124458312988, + "selected_utility_gain_over_base_at_8": -1.9307887554168701, "selector_regret_at_8": 1.9307887554168701, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9307887554168701 }, { + "base_success": 0.0, + "base_utility": 0.058033183217048645, "branch_car_at_8": 1.974869817495346, "chart_id": "ms-9fad82fd288e4296e354", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1815411965846294, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514, "seed": "1176943004/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02513018250465393, + "selected_utility_gain_over_base_at_8": -0.032903000712394714, "selector_regret_at_8": 1.974869817495346, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.974869817495346 }, { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.9512543678283691, "chart_id": "ms-b8bfdd3e6c31bd98082d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28991947553571784, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, "seed": "998761162/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04874563217163086, + "selected_utility_gain_over_base_at_8": -0.019070938229560852, "selector_regret_at_8": 1.9512543678283691, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9512543678283691 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9863876849412918, "chart_id": "ms-fc8084d5cd7da937fc02", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1306768270010748, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1991414747/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013612315058708191, + "selected_utility_gain_over_base_at_8": -1.9863876849412918, "selector_regret_at_8": 1.9863876849412918, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9863876849412918 }, { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.5863077491521835, "chart_id": "ms-3afd8959363e917591d4", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18559462278331035, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8128874897956848, + "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579, "seed": "75070641/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22657974064350128, + "selected_utility_gain_over_base_at_8": 0.02382810413837433, "selector_regret_at_8": 0.5863077491521835, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1871125102043152, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7734202593564987 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.8253512382507324, "chart_id": "ms-3ff1aedb72f65a182b58", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.318234334809661, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1055567618/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.17464876174926758, + "selected_utility_gain_over_base_at_8": -1.8253512382507324, "selector_regret_at_8": 1.8253512382507324, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8253512382507324 }, { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 0.6370042562484741, "chart_id": "ms-4048f8f6e047422e3c79", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07450750352404123, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8232162594795227, + "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314, "seed": "1055157354/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18621200323104858, + "selected_utility_gain_over_base_at_8": 0.009720906615257263, "selector_regret_at_8": 0.6370042562484741, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1767837405204773, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8137879967689514 }, { + "base_success": 0.0, + "base_utility": 0.19844311475753784, "branch_car_at_8": 0.03616611659526825, "chart_id": "ms-5b26c33feed6992386b1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28555947838938045, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.23248937726020813, + "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029, "seed": "1522272686/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.19632326066493988, + "selected_utility_gain_over_base_at_8": -0.0021198540925979614, "selector_regret_at_8": 0.03616611659526825, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7675106227397919, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8036767393350601 }, { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.6792904287576675, "chart_id": "ms-63ac245579b1188935ad", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07548939908426067, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8358442187309265, + "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215, "seed": "211245842/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15655378997325897, + "selected_utility_gain_over_base_at_8": -0.6822806149721146, "selector_regret_at_8": 0.6792904287576675, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1641557812690735, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.843446210026741 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-6c8d06bace98a28a8e4e", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1665739511/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.7044661641120911, "chart_id": "ms-72777024349c2aacd5e0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.03135306008748828, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8125, + "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748, "seed": "256770874/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10803383588790894, + "selected_utility_gain_over_base_at_8": -0.7477142214775085, "selector_regret_at_8": 0.7044661641120911, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1875, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.891966164112091 }, { + "base_success": 0.0, + "base_utility": 0.20372186601161957, "branch_car_at_8": 1.7757539749145508, "chart_id": "ms-7fc5696cb3dc7b9bb9f7", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07643021229779011, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804, "seed": "957101489/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22424602508544922, + "selected_utility_gain_over_base_at_8": 0.02052415907382965, "selector_regret_at_8": 1.7757539749145508, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7757539749145508 }, { + "base_success": 0.0, + "base_utility": 0.8431589603424072, "branch_car_at_8": 0.0, "chart_id": "ms-9a14552b7db695dee92c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13704637411112214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8532722592353821, + "proposal_oracle_utility_gain_over_base_at_8": 0.010113298892974854, "seed": "77147264/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8532722592353821, + "selected_utility_gain_over_base_at_8": 0.010113298892974854, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.146727740764618, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.146727740764618 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.6036040186882019, "chart_id": "ms-9f5d57fe2153fd267774", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18975917078230178, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8267073631286621, + "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338, "seed": "1607773860/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.2231033444404602, + "selected_utility_gain_over_base_at_8": -1.7768966555595398, "selector_regret_at_8": 0.6036040186882019, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.173292636871338, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7768966555595398 }, { + "base_success": 0.0, + "base_utility": 0.8324764966964722, "branch_car_at_8": 0.6467066705226898, "chart_id": "ms-b0fee26e1db967f90b81", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046694794435454434, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8275434374809265, + "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654, "seed": "1358421325/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1808367669582367, + "selected_utility_gain_over_base_at_8": -0.6516397297382355, "selector_regret_at_8": 0.6467066705226898, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.1724565625190735, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8191632330417633 }, { + "base_success": 0.0, + "base_utility": 0.8307622671127319, "branch_car_at_8": 0.0070628821849823, "chart_id": "ms-bc2d3b425d9163a62dbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14047863046133235, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22209535539150238, + "proposal_oracle_utility_gain_over_base_at_8": -0.6086669117212296, "seed": "1271718007/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21503247320652008, + "selected_utility_gain_over_base_at_8": -0.6157297939062119, "selector_regret_at_8": 0.0070628821849823, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7779046446084976, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.78496752679348 }, { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.7668711841106415, "chart_id": "ms-d1960368ef76bbb10669", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07978206783428046, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, "seed": "416183106/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23312881588935852, + "selected_utility_gain_over_base_at_8": 0.03280240297317505, "selector_regret_at_8": 1.7668711841106415, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.7668711841106415 }, { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, "chart_id": "ms-efdc07c985935d871adf", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.16443847051324711, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, "seed": "706479600/train1", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.0, "chart_id": "ms-fd2387aeed9bfc9a0d08", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13114443223224567, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.046244338154792786, + "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537, "seed": "1919198188/train1", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046244338154792786, + "selected_utility_gain_over_base_at_8": 0.009045466780662537, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9537556618452072, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9537556618452072 }, { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.5006794333457947, "chart_id": "ms-484ea653bbca2de3a4a2", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10207358838016511, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7278411984443665, + "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227, "seed": "1231780997/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22716176509857178, + "selected_utility_gain_over_base_at_8": -0.49984437227249146, "selector_regret_at_8": 0.5006794333457947, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2721588015556335, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.7728382349014282 }, { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 1.772506669163704, "chart_id": "ms-4b4cd12d70c0615e0930", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17388092747958847, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926, "seed": "1025740212/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22749333083629608, + "selected_utility_gain_over_base_at_8": -0.3008803278207779, "selector_regret_at_8": 1.772506669163704, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.772506669163704 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.28389310836792, "chart_id": "ms-6e268d1e2915740816a1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22214884767353088, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1475440973/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7161068916320801, + "selected_utility_gain_over_base_at_8": -1.28389310836792, "selector_regret_at_8": 1.28389310836792, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.28389310836792 }, { + "base_success": 0.0, + "base_utility": 0.7238867878913879, "branch_car_at_8": 1.273692786693573, "chart_id": "ms-8864796e31bbe93d7c5d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2702494805053396, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612, "seed": "551504635/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.726307213306427, + "selected_utility_gain_over_base_at_8": 0.0024204254150390625, "selector_regret_at_8": 1.273692786693573, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.273692786693573 }, { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.06608206033706665, "chart_id": "ms-b04b489f4fbad93b65ca", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.032532089835823584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7025249004364014, + "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276, "seed": "1113268940/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6364428400993347, + "selected_utility_gain_over_base_at_8": 0.21413031220436096, "selector_regret_at_8": 0.06608206033706665, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.2974750995635986, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.3635571599006653 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.280472755432129, "chart_id": "ms-c663f7e73e7e8376a05c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16399362968800954, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1528740417/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7195272445678711, + "selected_utility_gain_over_base_at_8": -1.280472755432129, "selector_regret_at_8": 1.280472755432129, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "LiftPegUpright-v1" + "task_id": "LiftPegUpright-v1", + "total_car_to_hidden_at_8": 1.280472755432129 }, { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.35906313359737396, "chart_id": "ms-2458fcfffb3f96c6ca1a", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, "seed": "1273247072/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, "selector_regret_at_8": 0.35906313359737396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.5362297892570496, - "task_id": "PickCube-v1" + "task_id": "PickCube-v1", + "total_car_to_hidden_at_8": 1.8952929228544235 }, { + "base_success": 0.0, + "base_utility": 0.050487834960222244, "branch_car_at_8": 0.007528863847255707, "chart_id": "ms-0345d8ac530f88cefd61", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1798098385334015, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.062358601136824246, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09839755296707153, + "proposal_oracle_utility_gain_over_base_at_8": 0.04790971800684929, "seed": "141984615/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09086868911981583, + "selected_utility_gain_over_base_at_8": 0.04038085415959358, "selector_regret_at_8": 0.007528863847255707, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.08141228556632996, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.08894114941358566 }, { + "base_success": 0.0, + "base_utility": 0.07743936777114868, "branch_car_at_8": 0.0, "chart_id": "ms-0819eb89e627595e7e1b", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.14572584629058838, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.06100934081266586, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.030317068099975586, + "proposal_oracle_utility_gain_over_base_at_8": -0.047122299671173096, "seed": "2015956957/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.030317068099975586, + "selected_utility_gain_over_base_at_8": -0.047122299671173096, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.11540877819061279, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.11540877819061279 }, { + "base_success": 0.0, + "base_utility": 0.19400137662887573, "branch_car_at_8": 0.02622777223587036, "chart_id": "ms-0dee962a5901fb8b5bbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15090002357562393, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11867360025644302, + "proposal_oracle_utility_gain_over_base_at_8": -0.07532777637243271, "seed": "565369392/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09244582802057266, + "selected_utility_gain_over_base_at_8": -0.10155554860830307, "selector_regret_at_8": 0.02622777223587036, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.881326399743557, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9075541719794273 }, { + "base_success": 0.0, + "base_utility": 0.04583283141255379, "branch_car_at_8": 0.07351336628198624, "chart_id": "ms-126eb8b36769f9ba4671", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1985912322998047, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1968725342775377, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.15246088802814484, + "proposal_oracle_utility_gain_over_base_at_8": 0.10662805661559105, "seed": "767747424/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0789475217461586, + "selected_utility_gain_over_base_at_8": 0.03311469033360481, "selector_regret_at_8": 0.07351336628198624, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.04613034427165985, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.11964371055364609 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-16661b2fd2c85f57a66c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1549324805/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002880295505747199, "chart_id": "ms-20e1efbb745fd02abc95", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.008822272371479922, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0008872548933140934, + "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686, "seed": "1375492786/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0005992253427393734, + "selected_utility_gain_over_base_at_8": -1.9994007746572606, "selector_regret_at_8": 0.0002880295505747199, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.24256371456431225, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24285174411488697 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-2dc5138251afd26a040d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "565621826/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.08252689987421036, "branch_car_at_8": 0.007421374320983887, "chart_id": "ms-387e6771a55be98f1a59", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20744429528713226, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11281705834695263, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.20954282581806183, + "proposal_oracle_utility_gain_over_base_at_8": 0.12701592594385147, "seed": "767747424/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.20212145149707794, + "selected_utility_gain_over_base_at_8": 0.11959455162286758, "selector_regret_at_8": 0.007421374320983887, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.005322843790054321 }, { + "base_success": 0.0, + "base_utility": 0.03917040303349495, "branch_car_at_8": 0.002210298553109169, "chart_id": "ms-38f7298184a2c5005e94", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1767096221446991, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17294274235303314, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.020024359226226807, + "proposal_oracle_utility_gain_over_base_at_8": -0.019146043807268143, "seed": "2015956957/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.017814060673117638, + "selected_utility_gain_over_base_at_8": -0.02135634236037731, "selector_regret_at_8": 0.002210298553109169, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1566852629184723, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15889556147158146 }, { + "base_success": 0.0, + "base_utility": 0.009124159812927246, "branch_car_at_8": 0.0006982283666729927, "chart_id": "ms-482c5c8d208b5907a2c2", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1834145486354828, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.026132711008509262, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.014287729747593403, + "proposal_oracle_utility_gain_over_base_at_8": 0.005163569934666157, "seed": "626486813/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01358950138092041, + "selected_utility_gain_over_base_at_8": 0.004465341567993164, "selector_regret_at_8": 0.0006982283666729927, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16912681888788939, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.16982504725456238 }, { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.034092940390110016, "chart_id": "ms-4da03300d92420e80ed0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08595818214364334, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12480536848306656, + "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645, "seed": "1747178030/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09071242809295654, + "selected_utility_gain_over_base_at_8": -0.05559416115283966, "selector_regret_at_8": 0.034092940390110016, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8751946315169334, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9092875719070435 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.006413221824914217, "chart_id": "ms-57be9ae181c077e7821f", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.4054335986359131, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012105385772883892, + "proposal_oracle_utility_gain_over_base_at_8": -1.987894614227116, "seed": "1549324805/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.005692163947969675, + "selected_utility_gain_over_base_at_8": -1.9943078360520303, "selector_regret_at_8": 0.006413221824914217, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.987894614227116, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9943078360520303 }, { + "base_success": 0.0, + "base_utility": 0.1544450968503952, "branch_car_at_8": 0.0, "chart_id": "ms-6388b5d2356c556c3e55", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1138946652254407, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8455549031496048, "seed": "2015956957/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8455549031496048, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.06173701956868172, "branch_car_at_8": 0.016335289925336838, "chart_id": "ms-638d2ba1917eff4d09e0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.11176212877035141, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1888708945281958, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07309168577194214, + "proposal_oracle_utility_gain_over_base_at_8": 0.011354666203260422, "seed": "767747424/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0567563958466053, + "selected_utility_gain_over_base_at_8": -0.004980623722076416, "selector_regret_at_8": 0.016335289925336838, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.03867044299840927, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.05500573292374611 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.001363794319331646, "chart_id": "ms-652ad552ba98d2736620", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2367449700832367, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07223239988980909, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.014465670101344585, + "proposal_oracle_utility_gain_over_base_at_8": -1.9855343298986554, "seed": "767747424/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.01310187578201294, + "selected_utility_gain_over_base_at_8": -1.986898124217987, "selector_regret_at_8": 0.001363794319331646, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.2222792999818921, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22364309430122375 }, { + "base_success": 0.0, + "base_utility": 0.03784912824630737, "branch_car_at_8": 0.012949071824550629, "chart_id": "ms-674fad0e7c042d23629e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.15573063492774963, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13958284884511013, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08471894264221191, + "proposal_oracle_utility_gain_over_base_at_8": 0.04686981439590454, "seed": "2015956957/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07176987081766129, + "selected_utility_gain_over_base_at_8": 0.03392074257135391, "selector_regret_at_8": 0.012949071824550629, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.07101169228553772, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.08396076411008835 }, { + "base_success": 0.0, + "base_utility": 0.12791956961154938, "branch_car_at_8": 0.010773517191410065, "chart_id": "ms-728b865677134d3788e3", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.22658847272396088, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12813704483770852, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0885184183716774, + "proposal_oracle_utility_gain_over_base_at_8": -0.03940115123987198, "seed": "750448296/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07774490118026733, + "selected_utility_gain_over_base_at_8": -0.05017466843128204, "selector_regret_at_8": 0.010773517191410065, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.13807005435228348, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.14884357154369354 }, { + "base_success": 0.0, + "base_utility": 0.20529556274414062, "branch_car_at_8": 0.05660251900553703, "chart_id": "ms-7ea98b8f4d2220347ee0", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.27989473938941956, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12566184669419236, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11846604198217392, + "proposal_oracle_utility_gain_over_base_at_8": -0.0868295207619667, "seed": "565369392/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06186352297663689, + "selected_utility_gain_over_base_at_8": -0.14343203976750374, "selector_regret_at_8": 0.05660251900553703, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.16142869740724564, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.21803121641278267 }, { + "base_success": 0.0, + "base_utility": 0.18213583528995514, "branch_car_at_8": 0.003586272243410349, "chart_id": "ms-7f57045c114ddad153b5", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.227622389793396, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22935436227797687, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.00681296968832612, + "proposal_oracle_utility_gain_over_base_at_8": -0.17532286560162902, "seed": "278008524/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0032266974449157715, + "selected_utility_gain_over_base_at_8": -0.17890913784503937, "selector_regret_at_8": 0.003586272243410349, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.22080942010506988, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22439569234848022 }, { + "base_success": 0.0, + "base_utility": 0.0056487321853637695, "branch_car_at_8": 0.001928985584527254, "chart_id": "ms-81f4e8858b2ba90aea2e", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1631626933813095, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1734428620262507, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.008817712776362896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0031689805909991264, "seed": "626486813/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006888727191835642, + "selected_utility_gain_over_base_at_8": 0.0012399950064718723, "selector_regret_at_8": 0.001928985584527254, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.1543449806049466, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.15627396618947387 }, { + "base_success": 0.0, + "base_utility": 0.00698717450723052, "branch_car_at_8": 0.08708053827285767, "chart_id": "ms-8a72b73ac452bc190aeb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.10329639911651611, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.5629214026808478, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.10641509294509888, + "proposal_oracle_utility_gain_over_base_at_8": 0.09942791843786836, "seed": "626486813/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.01933455467224121, + "selected_utility_gain_over_base_at_8": 0.01234738016501069, "selector_regret_at_8": 0.08708053827285767, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0839618444442749 }, { + "base_success": 0.0, + "base_utility": 0.08285123109817505, "branch_car_at_8": 0.032952308654785156, "chart_id": "ms-a96bca1ef0b74f686018", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.1572066992521286, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.34632898516451505, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06243818998336792, + "proposal_oracle_utility_gain_over_base_at_8": -0.02041304111480713, "seed": "141984615/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.029485881328582764, + "selected_utility_gain_over_base_at_8": -0.053365349769592285, "selector_regret_at_8": 0.032952308654785156, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.09476850926876068, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.12772081792354584 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.05388810485601425, "chart_id": "ms-ac4fb33f3ab4db853995", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.24514405918899088, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.16087166965007782, + "proposal_oracle_utility_gain_over_base_at_8": -1.8391283303499222, "seed": "565369392/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.10698356479406357, + "selected_utility_gain_over_base_at_8": -1.8930164352059364, "selector_regret_at_8": 0.05388810485601425, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8391283303499222, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8930164352059364 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-c31b98b3d3702c6f9c8c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "2054899126/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.037537217140197754, "branch_car_at_8": 0.03309768531471491, "chart_id": "ms-c832439976a37f52d094", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2594742774963379, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.34007124070568584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04554198309779167, + "proposal_oracle_utility_gain_over_base_at_8": 0.008004765957593918, "seed": "585904130/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012444297783076763, + "selected_utility_gain_over_base_at_8": -0.02509291935712099, "selector_regret_at_8": 0.03309768531471491, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.21393229439854622, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.24702997971326113 }, { + "base_success": 0.0, + "base_utility": 0.09138330072164536, "branch_car_at_8": 0.0, "chart_id": "ms-c96b415f083f60857334", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.26426148414611816, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2676934704052151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0532812662422657, + "proposal_oracle_utility_gain_over_base_at_8": -0.038102034479379654, "seed": "585904130/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0532812662422657, + "selected_utility_gain_over_base_at_8": -0.038102034479379654, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.21098021790385246, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.21098021790385246 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-d73fd5fce55fb58ef8e9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2870871533378466, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "534717141/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.14809735119342804, "branch_car_at_8": 0.0010104775428771973, "chart_id": "ms-da142025733b53fcf7eb", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.2643269896507263, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17434085305175193, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.001866757869720459, + "proposal_oracle_utility_gain_over_base_at_8": -0.14623059332370758, "seed": "141984615/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0008562803268432617, + "selected_utility_gain_over_base_at_8": -0.14724107086658478, "selector_regret_at_8": 0.0010104775428771973, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.26246023178100586, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.26347070932388306 }, { + "base_success": 0.0, + "base_utility": 0.1362423449754715, "branch_car_at_8": 0.03072514245286584, "chart_id": "ms-e2bad6e99e8a281207cc", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20825156569480896, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3105160115580491, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03793333098292351, + "proposal_oracle_utility_gain_over_base_at_8": -0.09830901399254799, "seed": "1549324805/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.007208188530057669, + "selected_utility_gain_over_base_at_8": -0.12903415644541383, "selector_regret_at_8": 0.03072514245286584, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.17031823471188545, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.2010433771647513 }, { + "base_success": 0.0, + "base_utility": 0.09695577621459961, "branch_car_at_8": 0.0, "chart_id": "ms-e30fd80adc089e9f697a", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18920521438121796, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17993304523315273, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13661859929561615, + "proposal_oracle_utility_gain_over_base_at_8": 0.03966282308101654, "seed": "585904130/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13661859929561615, + "selected_utility_gain_over_base_at_8": 0.03966282308101654, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.05258661508560181, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.05258661508560181 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.03718312829732895, "chart_id": "ms-e9ed5d488afdfd6449d1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17997058384239856, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.1506197303533554, + "proposal_oracle_utility_gain_over_base_at_8": -1.8493802696466446, "seed": "626486813/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.11343660205602646, + "selected_utility_gain_over_base_at_8": -1.8865633979439735, "selector_regret_at_8": 0.03718312829732895, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8493802696466446, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.8865633979439735 }, { + "base_success": 0.0, + "base_utility": 0.09514755010604858, "branch_car_at_8": 0.024643758311867714, "chart_id": "ms-ec013396e4aff6170ee1", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24502670764923096, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15485012388319694, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.04536725953221321, + "proposal_oracle_utility_gain_over_base_at_8": -0.04978029057383537, "seed": "750448296/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.020723501220345497, + "selected_utility_gain_over_base_at_8": -0.07442404888570309, "selector_regret_at_8": 0.024643758311867714, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.19965944811701775, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.22430320642888546 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.006491700187325478, "chart_id": "ms-ee30ebaf98622759d888", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12019361687075947, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.02436021901667118, + "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288, "seed": "636680137/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.017868518829345703, + "selected_utility_gain_over_base_at_8": -1.9821314811706543, "selector_regret_at_8": 0.006491700187325478, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9756397809833288, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 1.9821314811706543 }, { + "base_success": 0.0, + "base_utility": 0.026694953441619873, "branch_car_at_8": 0.012709278613328934, "chart_id": "ms-f3f1d07cfd59b823da47", + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.17124073207378387, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.22819684948441313, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06399381160736084, + "proposal_oracle_utility_gain_over_base_at_8": 0.03729885816574097, "seed": "2015956957/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.051284532994031906, + "selected_utility_gain_over_base_at_8": 0.024589579552412033, "selector_regret_at_8": 0.012709278613328934, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.10724692046642303, - "task_id": "PullCube-v1" + "task_id": "PullCube-v1", + "total_car_to_hidden_at_8": 0.11995619907975197 }, { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 1.861636370420456, "chart_id": "ms-081952a63087fc8f83cd", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09903289408743085, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419, "seed": "2141802816/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13836362957954407, + "selected_utility_gain_over_base_at_8": 0.06755638122558594, "selector_regret_at_8": 1.861636370420456, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.861636370420456 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-121bc2e6676c1ac10911", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "77147264/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-22f153f0d4fa656018e6", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.28406274395520603, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1271718007/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.024854734539985657, "chart_id": "ms-27c9f70e97b8850cc3ab", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2146835718150399, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.043482705950737, + "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201, "seed": "1881223807/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018627971410751343, + "selected_utility_gain_over_base_at_8": -0.013399019837379456, "selector_regret_at_8": 0.024854734539985657, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.956517294049263, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9813720285892487 }, { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, "chart_id": "ms-2867deaca7e98582620b", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, "seed": "1189392917/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, "chart_id": "ms-365d8c6de6de16f66744", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.14617959762414967, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, "seed": "1997090716/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03280790150165558, "branch_car_at_8": 0.036816492676734924, "chart_id": "ms-3c326d05fec66e1d6bea", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.08134846381937184, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05891832709312439, + "proposal_oracle_utility_gain_over_base_at_8": 0.02611042559146881, "seed": "278008524/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.022101834416389465, + "selected_utility_gain_over_base_at_8": -0.010706067085266113, "selector_regret_at_8": 0.036816492676734924, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9410816729068756, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9778981655836105 }, { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, "chart_id": "ms-4dbdc6ed61dccebbfe92", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, "seed": "794181626/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-58950fed04d203b97ab9", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.233029795067053, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1113371261/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9686058908700943, "chart_id": "ms-651bc0395d78d2819158", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.5995435922806452, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1483645336/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0313941091299057, + "selected_utility_gain_over_base_at_8": -1.9686058908700943, "selector_regret_at_8": 1.9686058908700943, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.9686058908700943 }, { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, "chart_id": "ms-9fad82fd288e4296e354", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17468938592004596, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, "seed": "1176943004/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.939187914133072, "chart_id": "ms-b8bfdd3e6c31bd98082d", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11706618974004251, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, "seed": "998761162/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0608120858669281, + "selected_utility_gain_over_base_at_8": -0.007004484534263611, "selector_regret_at_8": 1.939187914133072, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 1.939187914133072 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-fc8084d5cd7da937fc02", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11896663871153935, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1991414747/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "PushCube-v1" + "task_id": "PushCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.3710349500179291, "chart_id": "ms-3afd8959363e917591d4", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09051828052202912, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5650836229324341, + "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713, "seed": "75070641/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.194048672914505, + "selected_utility_gain_over_base_at_8": -0.008702963590621948, "selector_regret_at_8": 0.3710349500179291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.434916377067566, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.805951327085495 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9218642711639404, "chart_id": "ms-3ff1aedb72f65a182b58", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19289744165908876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1055567618/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07813572883605957, + "selected_utility_gain_over_base_at_8": -1.9218642711639404, "selector_regret_at_8": 1.9218642711639404, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9218642711639404 }, { + "base_success": 0.0, + "base_utility": 0.19648407399654388, "branch_car_at_8": 0.03480546176433563, "chart_id": "ms-4048f8f6e047422e3c79", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1805709977888097, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22467364370822906, + "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518, "seed": "1055157354/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18986818194389343, + "selected_utility_gain_over_base_at_8": -0.006615892052650452, "selector_regret_at_8": 0.03480546176433563, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.775326356291771, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8101318180561066 }, { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.017972871661186218, "chart_id": "ms-5b26c33feed6992386b1", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.3276128524355476, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1949000507593155, + "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294, "seed": "1522272686/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.17692717909812927, + "selected_utility_gain_over_base_at_8": -0.01783391833305359, "selector_regret_at_8": 0.017972871661186218, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.8050999492406845, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8230728209018707 }, { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.0, "chart_id": "ms-63ac245579b1188935ad", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13508468824583414, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265, "seed": "211245842/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1611655950546265, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-6c8d06bace98a28a8e4e", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2508232858898205, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1665739511/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.046216949820518494, "chart_id": "ms-72777024349c2aacd5e0", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046630940401554466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.21061499416828156, + "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359, "seed": "256770874/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16439804434776306, + "selected_utility_gain_over_base_at_8": -0.6913500130176544, "selector_regret_at_8": 0.046216949820518494, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.7893850058317184, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.835601955652237 }, { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 1.432861328125, "chart_id": "ms-7fc5696cb3dc7b9bb9f7", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.21219243624964876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252, "seed": "957101489/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.567138671875, + "selected_utility_gain_over_base_at_8": 0.3631737232208252, "selector_regret_at_8": 1.432861328125, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.432861328125 }, { + "base_success": 0.0, + "base_utility": 0.8431589603424072, "branch_car_at_8": 0.0, "chart_id": "ms-9a14552b7db695dee92c", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.24618369758422665, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1568410396575928, "seed": "77147264/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1568410396575928, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, "chart_id": "ms-9f5d57fe2153fd267774", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.24177237819615016, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, "seed": "1607773860/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 1.90096914768219, "chart_id": "ms-b0fee26e1db967f90b81", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.32948159974174274, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187, "seed": "1358421325/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09903085231781006, + "selected_utility_gain_over_base_at_8": -0.7336714267730713, "selector_regret_at_8": 1.90096914768219, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.90096914768219 }, { + "base_success": 0.0, + "base_utility": 0.8307622671127319, "branch_car_at_8": 0.39469505846500397, "chart_id": "ms-bc2d3b425d9163a62dbb", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18608118833691456, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5271345376968384, + "proposal_oracle_utility_gain_over_base_at_8": -0.30362772941589355, "seed": "1271718007/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1324394792318344, + "selected_utility_gain_over_base_at_8": -0.6983227878808975, "selector_regret_at_8": 0.39469505846500397, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.4728654623031616, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.8675605207681656 }, { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.812951534986496, "chart_id": "ms-d1960368ef76bbb10669", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18072825209943494, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, "seed": "416183106/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18704846501350403, + "selected_utility_gain_over_base_at_8": -0.013277947902679443, "selector_regret_at_8": 1.812951534986496, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.812951534986496 }, { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, "chart_id": "ms-efdc07c985935d871adf", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.22468613298280815, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, "seed": "706479600/train2", + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, "support_gap_at_8": 0.0, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 0.0 }, { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.018839523196220398, "chart_id": "ms-fd2387aeed9bfc9a0d08", + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "mode": "measured", "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1669540555093941, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06275103986263275, + "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502, "seed": "1919198188/train2", + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043911516666412354, + "selected_utility_gain_over_base_at_8": 0.0067126452922821045, "selector_regret_at_8": 0.018839523196220398, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, "support_gap_at_8": 1.9372489601373672, - "task_id": "StackCube-v1" + "task_id": "StackCube-v1", + "total_car_to_hidden_at_8": 1.9560884833335876 } ], "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39775413711583923, + "low": 0.25236406619385343, + "mean": 0.322695035460993, + "n": 141, + "num_samples": 500 + }, + "mean": 0.322695035460993, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 0.0, + "1176943004/train1": 0.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 0.0, + "1231780997/train1": 0.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.5, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 0.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.6666666666666666, + "1549324805/train2": 0.6666666666666666, + "1607773860/train0": 1.0, + "1607773860/train1": 1.0, + "1607773860/train2": 1.0, + "1665739511/train0": 1.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 1.0, + "1991414747/train1": 1.0, + "1991414747/train2": 1.0, + "1997090716/train0": 0.0, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.0, + "2015956957/train1": 0.0, + "2015956957/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 0.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 1.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 0.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.3333333333333333, + "565369392/train2": 0.3333333333333333, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 0.0, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25, + "77147264/train0": 0.5, + "77147264/train1": 0.5, + "77147264/train2": 0.5, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 0.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 0.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3306184012066365, + "low": 0.10666666666666666, + "mean": 0.23728506787330317, + "n": 5, + "num_samples": 500 + }, + "mean": 0.23728506787330317, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.358974358974359, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.34782608695652173, + "low": 0.22705314009661837, + "mean": 0.28502415458937197, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9718880789716623, + "low": 0.7188909860138495, + "mean": 0.84238802050422, + "n": 141, + "num_samples": 500 + }, + "mean": 0.84238802050422, + "per_seed": { + "1025740212/train0": 0.528373658657074, + "1025740212/train1": 0.528373658657074, + "1025740212/train2": 0.528373658657074, + "1055157354/train0": 0.17649109661579132, + "1055157354/train1": 0.17649109661579132, + "1055157354/train2": 0.19648407399654388, + "1055567618/train0": 2.0, + "1055567618/train1": 2.0, + "1055567618/train2": 2.0, + "1113268940/train0": 0.42231252789497375, + "1113268940/train1": 0.42231252789497375, + "1113268940/train2": 0.42231252789497375, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 0.05094052851200104, + "1176943004/train1": 0.058033183217048645, + "1176943004/train2": 0.05094052851200104, + "1189392917/train0": 0.10841864347457886, + "1189392917/train1": 0.10841864347457886, + "1189392917/train2": 0.10841864347457886, + "1231780997/train0": 0.7270061373710632, + "1231780997/train1": 0.7270061373710632, + "1231780997/train2": 0.7270061373710632, + "1271718007/train0": 1.4142604768276215, + "1271718007/train1": 1.415381133556366, + "1271718007/train2": 1.415381133556366, + "1273247072/train0": 0.15342965722084045, + "1273247072/train1": 0.15342965722084045, + "1273247072/train2": 0.15342965722084045, + "1358421325/train0": 0.8327022790908813, + "1358421325/train1": 0.8324764966964722, + "1358421325/train2": 0.8327022790908813, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.09381213908394177, + "141984615/train1": 0.09381213908394177, + "141984615/train2": 0.09381213908394177, + "1475440973/train0": 2.0, + "1475440973/train1": 2.0, + "1475440973/train2": 2.0, + "1483645336/train0": 0.01641605794429779, + "1483645336/train1": 2.0, + "1483645336/train2": 2.0, + "1522272686/train0": 0.19476109743118286, + "1522272686/train1": 0.19844311475753784, + "1522272686/train2": 0.19476109743118286, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 2.0, + "1549324805/train0": 1.3787474483251572, + "1549324805/train1": 1.3787474483251572, + "1549324805/train2": 1.3787474483251572, + "1607773860/train0": 2.0, + "1607773860/train1": 2.0, + "1607773860/train2": 2.0, + "1665739511/train0": 2.0, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 0.1463065892457962, + "1747178030/train1": 0.1463065892457962, + "1747178030/train2": 0.1463065892457962, + "1881223807/train0": 0.0320269912481308, + "1881223807/train1": 0.0320269912481308, + "1881223807/train2": 0.0320269912481308, + "1919198188/train0": 0.037198856472969055, + "1919198188/train1": 0.03719887137413025, + "1919198188/train2": 0.03719887137413025, + "1991414747/train0": 2.0, + "1991414747/train1": 2.0, + "1991414747/train2": 2.0, + "1997090716/train0": 0.13927479088306427, + "1997090716/train1": 0.13927479088306427, + "1997090716/train2": 0.13927479088306427, + "2015956957/train0": 0.06711978986859321, + "2015956957/train1": 0.06711978986859321, + "2015956957/train2": 0.06711978986859321, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 0.8388344049453735, + "211245842/train1": 0.8388344049453735, + "211245842/train2": 0.8388344049453735, + "2141802816/train0": 0.07080724835395813, + "2141802816/train1": 0.07080724835395813, + "2141802816/train2": 0.07080724835395813, + "256770874/train0": 0.8557480573654175, + "256770874/train1": 0.8557480573654175, + "256770874/train2": 0.8557480573654175, + "278008524/train0": 0.10747186839580536, + "278008524/train1": 0.10747186839580536, + "278008524/train2": 0.10747186839580536, + "416183106/train0": 0.20032641291618347, + "416183106/train1": 0.20032641291618347, + "416183106/train2": 0.20032641291618347, + "534717141/train0": 2.0, + "534717141/train1": 2.0, + "534717141/train2": 2.0, + "551504635/train0": 0.7239725589752197, + "551504635/train1": 0.7247228622436523, + "551504635/train2": 0.7238867878913879, + "565369392/train0": 0.7997656464576721, + "565369392/train1": 0.7997656464576721, + "565369392/train2": 0.7997656464576721, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.07529209802548091, + "585904130/train1": 0.07529209802548091, + "585904130/train2": 0.07529209802548091, + "626486813/train0": 0.5054400166263804, + "626486813/train1": 0.5054400166263804, + "626486813/train2": 0.5054400166263804, + "636680137/train0": 2.0, + "636680137/train1": 2.0, + "636680137/train2": 2.0, + "706479600/train0": 0.8125406503677368, + "706479600/train1": 0.8350887894630432, + "706479600/train2": 0.8350887894630432, + "750448296/train0": 0.11153355985879898, + "750448296/train1": 0.11153355985879898, + "750448296/train2": 0.11153355985879898, + "75070641/train0": 0.20275163650512695, + "75070641/train1": 0.20275163650512695, + "75070641/train2": 0.20275163650512695, + "767747424/train0": 0.5475241877138615, + "767747424/train1": 0.5475241877138615, + "767747424/train2": 0.5475241877138615, + "77147264/train0": 1.4215794801712036, + "77147264/train1": 1.4215794801712036, + "77147264/train2": 1.4215794801712036, + "794181626/train0": 0.1733371466398239, + "794181626/train1": 0.18095605075359344, + "794181626/train2": 0.18095605075359344, + "957101489/train0": 0.2039649486541748, + "957101489/train1": 0.20372186601161957, + "957101489/train2": 0.2039649486541748, + "998761162/train0": 0.0689050555229187, + "998761162/train1": 0.06781657040119171, + "998761162/train2": 0.06781657040119171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9246198772067178, + "low": 0.4069265285873006, + "mean": 0.6891968629765195, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6891968629765195, + "per_task": { + "LiftPegUpright-v1": 1.0669810656044219, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.7709262684369699, + "StackCube-v1": 0.8034174028370116 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.851249745030154, + "low": 0.6279041040598771, + "mean": 0.7358034186915535, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, "branch_car_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5194647902708409, - "low": 0.2999603762294949, + "high": 0.5224681463797662, + "low": 0.2913920145155972, "mean": 0.40561739670197494, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.40561739670197494, "per_seed": { @@ -2441,11 +6121,11 @@ "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.5319262530406317, - "low": 0.19367229081819662, + "high": 0.5390796998678109, + "low": 0.18723032816778867, "mean": 0.3798908010369334, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.3798908010369334, "per_task": { @@ -2459,11 +6139,379 @@ "metric": "branch_car_at_8", "micro": { "confidence": 0.95, - "high": 0.3721372773162211, - "low": 0.21313571063115527, + "high": 0.37381450664971905, + "low": 0.2099735496069669, "mean": 0.293583450403069, "n": 207, - "num_samples": 1000 + "num_samples": 500 + }, + "n": 207 + }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8884160756501184, + "low": 0.7735224586288418, + "mean": 0.8358156028368796, + "n": 141, + "num_samples": 500 + }, + "mean": 0.8358156028368796, + "per_seed": { + "1025740212/train0": 1.0, + "1025740212/train1": 1.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 1.0, + "1055157354/train2": 1.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 1.0, + "1113268940/train1": 1.0, + "1113268940/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 1.0, + "1176943004/train1": 1.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 1.0, + "1231780997/train1": 1.0, + "1231780997/train2": 1.0, + "1271718007/train0": 1.0, + "1271718007/train1": 1.0, + "1271718007/train2": 1.0, + "1273247072/train0": 1.0, + "1273247072/train1": 1.0, + "1273247072/train2": 1.0, + "1358421325/train0": 1.0, + "1358421325/train1": 1.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 1.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.6666666666666666, + "1549324805/train2": 0.6666666666666666, + "1607773860/train0": 1.0, + "1607773860/train1": 1.0, + "1607773860/train2": 1.0, + "1665739511/train0": 1.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1881223807/train0": 1.0, + "1881223807/train1": 1.0, + "1881223807/train2": 1.0, + "1919198188/train0": 1.0, + "1919198188/train1": 1.0, + "1919198188/train2": 1.0, + "1991414747/train0": 1.0, + "1991414747/train1": 1.0, + "1991414747/train2": 1.0, + "1997090716/train0": 1.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.2, + "2015956957/train2": 0.2, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "2141802816/train0": 1.0, + "2141802816/train1": 1.0, + "2141802816/train2": 1.0, + "256770874/train0": 1.0, + "256770874/train1": 1.0, + "256770874/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "416183106/train0": 1.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 1.0, + "551504635/train0": 1.0, + "551504635/train1": 1.0, + "551504635/train2": 1.0, + "565369392/train0": 0.6666666666666666, + "565369392/train1": 0.6666666666666666, + "565369392/train2": 0.6666666666666666, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 1.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 1.0, + "75070641/train1": 1.0, + "75070641/train2": 1.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 1.0, + "77147264/train2": 1.0, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 1.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 1.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.7246376811594203, + "low": 0.6086956521739131, + "mean": 0.6666666666666666, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8005860548316728, + "low": 1.5936215176297697, + "mean": 1.706202811163896, + "n": 141, + "num_samples": 500 + }, + "mean": 1.706202811163896, + "per_seed": { + "1025740212/train0": 2.0, + "1025740212/train1": 2.0, + "1025740212/train2": 2.0, + "1055157354/train0": 2.0, + "1055157354/train1": 2.0, + "1055157354/train2": 2.0, + "1055567618/train0": 2.0, + "1055567618/train1": 2.0, + "1055567618/train2": 2.0, + "1113268940/train0": 2.0, + "1113268940/train1": 2.0, + "1113268940/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 2.0, + "1176943004/train1": 2.0, + "1176943004/train2": 2.0, + "1189392917/train0": 2.0, + "1189392917/train1": 2.0, + "1189392917/train2": 2.0, + "1231780997/train0": 2.0, + "1231780997/train1": 2.0, + "1231780997/train2": 2.0, + "1271718007/train0": 2.0, + "1271718007/train1": 2.0, + "1271718007/train2": 2.0, + "1273247072/train0": 2.0, + "1273247072/train1": 2.0, + "1273247072/train2": 2.0, + "1358421325/train0": 2.0, + "1358421325/train1": 2.0, + "1358421325/train2": 2.0, + "1375492786/train0": 0.24345096945762634, + "1375492786/train1": 0.24345096945762634, + "1375492786/train2": 0.24345096945762634, + "141984615/train0": 0.20044784247875214, + "141984615/train1": 0.20044784247875214, + "141984615/train2": 0.20044784247875214, + "1475440973/train0": 2.0, + "1475440973/train1": 2.0, + "1475440973/train2": 2.0, + "1483645336/train0": 2.0, + "1483645336/train1": 2.0, + "1483645336/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 2.0, + "1549324805/train0": 1.4027505218982697, + "1549324805/train1": 1.4027505218982697, + "1549324805/train2": 1.4027505218982697, + "1607773860/train0": 2.0, + "1607773860/train1": 2.0, + "1607773860/train2": 2.0, + "1665739511/train0": 2.0, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1881223807/train0": 2.0, + "1881223807/train1": 2.0, + "1881223807/train2": 2.0, + "1919198188/train0": 2.0, + "1919198188/train1": 2.0, + "1919198188/train2": 2.0, + "1991414747/train0": 2.0, + "1991414747/train1": 2.0, + "1991414747/train2": 2.0, + "1997090716/train0": 2.0, + "1997090716/train1": 2.0, + "1997090716/train2": 2.0, + "2015956957/train0": 0.5298813670873642, + "2015956957/train1": 0.5298813670873642, + "2015956957/train2": 0.5298813670873642, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "2141802816/train0": 2.0, + "2141802816/train1": 2.0, + "2141802816/train2": 2.0, + "256770874/train0": 2.0, + "256770874/train1": 2.0, + "256770874/train2": 2.0, + "278008524/train0": 1.113811194896698, + "278008524/train1": 1.113811194896698, + "278008524/train2": 1.113811194896698, + "416183106/train0": 2.0, + "416183106/train1": 2.0, + "416183106/train2": 2.0, + "534717141/train0": 2.0, + "534717141/train1": 2.0, + "534717141/train2": 2.0, + "551504635/train0": 2.0, + "551504635/train1": 2.0, + "551504635/train2": 2.0, + "565369392/train0": 1.4266315797964733, + "565369392/train1": 1.4266315797964733, + "565369392/train2": 1.4266315797964733, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.23764699200789133, + "585904130/train1": 0.23764699200789133, + "585904130/train2": 0.23764699200789133, + "626486813/train0": 0.6124684102833271, + "626486813/train1": 0.6124684102833271, + "626486813/train2": 0.6124684102833271, + "636680137/train0": 2.0, + "636680137/train1": 2.0, + "636680137/train2": 2.0, + "706479600/train0": 2.0, + "706479600/train1": 2.0, + "706479600/train2": 2.0, + "750448296/train0": 0.23580759018659592, + "750448296/train1": 0.23580759018659592, + "750448296/train2": 0.23580759018659592, + "75070641/train0": 2.0, + "75070641/train1": 2.0, + "75070641/train2": 2.0, + "767747424/train0": 0.18863565661013126, + "767747424/train1": 0.18863565661013126, + "767747424/train2": 0.18863565661013126, + "77147264/train0": 2.0, + "77147264/train1": 2.0, + "77147264/train2": 2.0, + "794181626/train0": 2.0, + "794181626/train1": 2.0, + "794181626/train2": 2.0, + "957101489/train0": 2.0, + "957101489/train1": 2.0, + "957101489/train2": 2.0, + "998761162/train0": 2.0, + "998761162/train1": 2.0, + "998761162/train2": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.5056910838409898, + "low": 1.293966528766109, + "mean": 1.3999846732054932, + "n": 207, + "num_samples": 500 }, "n": 207 }, @@ -2471,11 +6519,11 @@ "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5674940898345152, - "low": 0.4113475177304965, + "high": 0.5641843971631206, + "low": 0.4092198581560284, "mean": 0.4849881796690307, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.4849881796690307, "per_seed": { @@ -2626,10 +6674,10 @@ "ci": { "confidence": 0.95, "high": 0.6328205128205128, - "low": 0.4294117647058823, + "low": 0.42745098039215684, "mean": 0.5411161387631975, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.5411161387631975, "per_task": { @@ -2643,11 +6691,11 @@ "metric": "outcome_ptr_at_8", "micro": { "confidence": 0.95, - "high": 0.5265700483091788, + "high": 0.5314009661835749, "low": 0.391304347826087, "mean": 0.45893719806763283, "n": 207, - "num_samples": 1000 + "num_samples": 500 }, "n": 207 }, @@ -2655,11 +6703,11 @@ "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.17446188421793687, - "low": 0.14114248449490324, + "high": 0.1737242314731193, + "low": 0.1410392771878472, "mean": 0.15793930902529071, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.15793930902529071, "per_seed": { @@ -2809,11 +6857,11 @@ "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.17042242389065887, - "low": 0.1574042611063619, + "high": 0.17033533868825532, + "low": 0.15683265673150842, "mean": 0.16359339232485098, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.16359339232485098, "per_task": { @@ -2827,195 +6875,2219 @@ "metric": "pairwise_causal_calibration_ece", "micro": { "confidence": 0.95, - "high": 0.1771431506285629, - "low": 0.14633605302901057, + "high": 0.1771397879569765, + "low": 0.14678388743440304, "mean": 0.16176667684993604, "n": 207, - "num_samples": 1000 + "num_samples": 500 }, "n": 207 }, - "selector_regret_at_8": { + "proposal_oracle_success_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5194647902708409, - "low": 0.2999603762294949, - "mean": 0.40561739670197494, + "high": 0.5497635933806146, + "low": 0.38581560283687943, + "mean": 0.47565011820330966, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, - "mean": 0.40561739670197494, + "mean": 0.47565011820330966, "per_seed": { - "1025740212/train0": 0.26188503205776215, - "1025740212/train1": 0.47739143669605255, - "1025740212/train2": 1.772506669163704, - "1055157354/train0": 1.8516457676887512, - "1055157354/train1": 0.6370042562484741, - "1055157354/train2": 0.03480546176433563, - "1055567618/train0": 1.183156132698059, - "1055567618/train1": 1.8253512382507324, - "1055567618/train2": 1.9218642711639404, - "1113268940/train0": 0.38470593094825745, - "1113268940/train1": 0.18306860327720642, - "1113268940/train2": 0.06608206033706665, - "1113371261/train0": 0.0, - "1113371261/train1": 0.0, - "1113371261/train2": 0.0, - "1176943004/train0": 0.0, - "1176943004/train1": 1.974869817495346, - "1176943004/train2": 0.0, - "1189392917/train0": 0.0, - "1189392917/train1": 0.0, - "1189392917/train2": 0.0, - "1231780997/train0": 0.4897426515817642, - "1231780997/train1": 0.0, - "1231780997/train2": 0.5006794333457947, - "1271718007/train0": 0.0, - "1271718007/train1": 0.00353144109249115, - "1271718007/train2": 0.19734752923250198, + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 1.0, + "1176943004/train1": 1.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 1.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, "1273247072/train0": 0.0, - "1273247072/train1": 0.45726464316248894, - "1273247072/train2": 0.35906313359737396, - "1358421325/train0": 0.5775325745344162, - "1358421325/train1": 0.6467066705226898, - "1358421325/train2": 1.90096914768219, - "1375492786/train0": 0.0002854267368093133, - "1375492786/train1": 0.00047226750757545233, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 1.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 1.0, + "1607773860/train1": 0.0, + "1607773860/train2": 1.0, + "1665739511/train0": 0.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 1.0, + "1991414747/train1": 1.0, + "1991414747/train2": 1.0, + "1997090716/train0": 1.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 1.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 1.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 1.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.0, + "565369392/train2": 0.0, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "706479600/train0": 1.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 0.5, + "77147264/train2": 1.0, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 0.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5987933634992458, + "low": 0.14901960784313725, + "mean": 0.3799899446958271, + "n": 5, + "num_samples": 500 + }, + "mean": 0.3799899446958271, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 0.0, + "PullCube-v1": 0.18627450980392157, + "PushCube-v1": 0.7692307692307693, + "StackCube-v1": 0.4444444444444444 + } + }, + "metric": "proposal_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.4444444444444444, + "low": 0.30917874396135264, + "mean": 0.37681159420289856, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "proposal_oracle_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2335697399527187, + "low": 0.06122931442080378, + "mean": 0.15295508274231676, + "n": 141, + "num_samples": 500 + }, + "mean": 0.15295508274231676, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.0, + "1176943004/train1": 1.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.5, + "1271718007/train1": 0.0, + "1271718007/train2": 0.0, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 1.0, + "1375492786/train0": -1.0, + "1375492786/train1": -1.0, + "1375492786/train2": -1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 1.0, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": -0.3333333333333333, + "1549324805/train2": -0.3333333333333333, + "1607773860/train0": 0.0, + "1607773860/train1": -1.0, + "1607773860/train2": 0.0, + "1665739511/train0": -1.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 0.0, + "1991414747/train1": 0.0, + "1991414747/train2": 0.0, + "1997090716/train0": 1.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 1.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 1.0, + "565369392/train0": 0.0, + "565369392/train1": -0.3333333333333333, + "565369392/train2": -0.3333333333333333, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": -0.25, + "626486813/train2": -0.25, + "636680137/train0": -1.0, + "636680137/train1": -1.0, + "636680137/train2": -1.0, + "706479600/train0": 1.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": -0.25, + "767747424/train2": -0.25, + "77147264/train0": 0.5, + "77147264/train1": 0.0, + "77147264/train2": 0.5, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 0.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3107692307692308, + "low": -0.01581699346405229, + "mean": 0.1427048768225239, + "n": 5, + "num_samples": 500 + }, + "mean": 0.1427048768225239, + "per_task": { + "LiftPegUpright-v1": 0.16666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": -0.10784313725490197, + "PushCube-v1": 0.41025641025641024, + "StackCube-v1": 0.24444444444444444 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.15942028985507245, + "low": 0.01932367149758454, + "mean": 0.09178743961352658, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2280026059425997, + "low": 0.9486234303862167, + "mean": 1.1034544456804372, + "n": 141, + "num_samples": 500 + }, + "mean": 1.1034544456804372, + "per_seed": { + "1025740212/train0": 0.4761262834072113, + "1025740212/train1": 0.6983401775360107, + "1025740212/train2": 2.0, + "1055157354/train0": 2.0, + "1055157354/train1": 0.8232162594795227, + "1055157354/train2": 0.22467364370822906, + "1055567618/train0": 2.0, + "1055567618/train1": 2.0, + "1055567618/train2": 2.0, + "1113268940/train0": 0.6646121144294739, + "1113268940/train1": 0.6617497801780701, + "1113268940/train2": 0.7025249004364014, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 2.0, + "1176943004/train1": 2.0, + "1176943004/train2": 2.0, + "1189392917/train0": 2.0, + "1189392917/train1": 2.0, + "1189392917/train2": 2.0, + "1231780997/train0": 0.7187411189079285, + "1231780997/train1": 2.0, + "1231780997/train2": 0.7278411984443665, + "1271718007/train0": 2.0, + "1271718007/train1": 1.1110476776957512, + "1271718007/train2": 1.2635672688484192, + "1273247072/train0": 0.13278456032276154, + "1273247072/train1": 0.4997493326663971, + "1273247072/train2": 0.46377021074295044, + "1358421325/train0": 0.8142459392547607, + "1358421325/train1": 0.8275434374809265, + "1358421325/train2": 2.0, + "1375492786/train0": 0.0016440550098195672, + "1375492786/train1": 0.0017900466918945312, + "1375492786/train2": 0.0008872548933140934, + "141984615/train0": 0.0797463705142339, + "141984615/train1": 0.09240151196718216, + "141984615/train2": 0.054234166940053306, + "1475440973/train0": 2.0, + "1475440973/train1": 2.0, + "1475440973/train2": 2.0, + "1483645336/train0": 2.0, + "1483645336/train1": 2.0, + "1483645336/train2": 2.0, + "1522272686/train0": 0.2435554713010788, + "1522272686/train1": 0.23248937726020813, + "1522272686/train2": 0.1949000507593155, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 2.0, + "1549324805/train0": 1.3427665165315072, + "1549324805/train1": 0.6785067518552145, + "1549324805/train2": 0.6833462389186025, + "1607773860/train0": 2.0, + "1607773860/train1": 0.8267073631286621, + "1607773860/train2": 2.0, + "1665739511/train0": 0.8216220736503601, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 0.11134227365255356, + "1747178030/train2": 0.12480536848306656, + "1881223807/train0": 0.0513521283864975, + "1881223807/train1": 0.034886181354522705, + "1881223807/train2": 0.043482705950737, + "1919198188/train0": 0.08992795646190643, + "1919198188/train1": 0.046244338154792786, + "1919198188/train2": 0.06275103986263275, + "1991414747/train0": 2.0, + "1991414747/train1": 2.0, + "1991414747/train2": 2.0, + "1997090716/train0": 2.0, + "1997090716/train1": 2.0, + "1997090716/train2": 2.0, + "2015956957/train0": 0.45124312303960323, + "2015956957/train1": 0.05629209652543068, + "2015956957/train2": 0.43981083631515505, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 0.8130142092704773, + "211245842/train1": 0.8358442187309265, + "211245842/train2": 2.0, + "2141802816/train0": 0.12472955882549286, + "2141802816/train1": 0.09589323401451111, + "2141802816/train2": 2.0, + "256770874/train0": 0.24107472598552704, + "256770874/train1": 0.8125, + "256770874/train2": 0.21061499416828156, + "278008524/train0": 0.011432985425926745, + "278008524/train1": 0.03147180867381394, + "278008524/train2": 0.032865648390725255, + "416183106/train0": 0.5778142213821411, + "416183106/train1": 2.0, + "416183106/train2": 2.0, + "534717141/train0": 2.0, + "534717141/train1": 2.0, + "534717141/train2": 2.0, + "551504635/train0": 0.7250897884368896, + "551504635/train1": 0.7224790453910828, + "551504635/train2": 2.0, + "565369392/train0": 0.7852714310089747, + "565369392/train1": 0.17366480827331543, + "565369392/train2": 0.1326704372962316, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.05598785417775313, + "585904130/train1": 0.032918572425842285, + "585904130/train2": 0.07848061621189117, + "626486813/train0": 0.5615248780231923, + "626486813/train1": 0.07365635968744755, + "626486813/train2": 0.07003506645560265, + "636680137/train0": 0.012414653785526752, + "636680137/train1": 0.013928472995758057, + "636680137/train2": 0.02436021901667118, + "706479600/train0": 2.0, + "706479600/train1": 2.0, + "706479600/train2": 2.0, + "750448296/train0": 0.1069931760430336, + "750448296/train1": 0.07342664152383804, + "750448296/train2": 0.0669428389519453, + "75070641/train0": 0.2396746575832367, + "75070641/train1": 0.8128874897956848, + "75070641/train2": 0.5650836229324341, + "767747424/train0": 0.5703969299793243, + "767747424/train1": 0.11342880013398826, + "767747424/train2": 0.11239026742987335, + "77147264/train0": 2.0, + "77147264/train1": 1.426636129617691, + "77147264/train2": 2.0, + "794181626/train0": 2.0, + "794181626/train1": 2.0, + "794181626/train2": 2.0, + "957101489/train0": 0.5881698727607727, + "957101489/train1": 2.0, + "957101489/train2": 2.0, + "998761162/train0": 0.08403940498828888, + "998761162/train1": 2.0, + "998761162/train2": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3557187332811518, + "low": 0.552532447482158, + "mean": 0.971318265720506, + "n": 5, + "num_samples": 500 + }, + "mean": 0.971318265720506, + "per_task": { + "LiftPegUpright-v1": 1.3387502448426352, + "PickCube-v1": 0.3654347012440364, + "PullCube-v1": 0.4324846717342963, + "PushCube-v1": 1.5530982193274376, + "StackCube-v1": 1.1668234914541245 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.0042415133076787, + "low": 0.758569956281625, + "mean": 0.8810875975700648, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.40175874303166603, + "low": 0.10017959579071863, + "mean": 0.2610664251762173, + "n": 141, + "num_samples": 500 + }, + "mean": 0.2610664251762173, + "per_seed": { + "1025740212/train0": -0.05224737524986267, + "1025740212/train1": 0.16996651887893677, + "1025740212/train2": 1.471626341342926, + "1055157354/train0": 1.8235089033842087, + "1055157354/train1": 0.6467251628637314, + "1055157354/train2": 0.02818956971168518, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 0.24229958653450012, + "1113268940/train1": 0.2394372522830963, + "1113268940/train2": 0.2802123725414276, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.949059471487999, + "1176943004/train1": 1.9419668167829514, + "1176943004/train2": 1.949059471487999, + "1189392917/train0": 1.8915813565254211, + "1189392917/train1": 1.8915813565254211, + "1189392917/train2": 1.8915813565254211, + "1231780997/train0": -0.008265018463134766, + "1231780997/train1": 1.2729938626289368, + "1231780997/train2": 0.0008350610733032227, + "1271718007/train0": 0.5857395231723785, + "1271718007/train1": -0.3043334558606148, + "1271718007/train2": -0.15181386470794678, + "1273247072/train0": -0.02064509689807892, + "1273247072/train1": 0.34631967544555664, + "1273247072/train2": 0.31034055352211, + "1358421325/train0": -0.018456339836120605, + "1358421325/train1": -0.004933059215545654, + "1358421325/train2": 1.1672977209091187, + "1375492786/train0": -1.9983559449901804, + "1375492786/train1": -1.9982099533081055, + "1375492786/train2": -1.999112745106686, + "141984615/train0": -0.01406576856970787, + "141984615/train1": -0.001410627116759618, + "141984615/train2": -0.039577972143888474, + "1475440973/train0": 0.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 1.9835839420557022, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 0.048794373869895935, + "1522272686/train1": 0.03404626250267029, + "1522272686/train2": 0.0001389533281326294, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 0.0, + "1549324805/train0": -0.035980931793649994, + "1549324805/train1": -0.7002406964699427, + "1549324805/train2": -0.6954012094065547, + "1607773860/train0": 0.0, + "1607773860/train1": -1.173292636871338, + "1607773860/train2": 0.0, + "1665739511/train0": -1.17837792634964, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.8536934107542038, + "1747178030/train1": -0.034964315593242645, + "1747178030/train2": -0.021501220762729645, + "1881223807/train0": 0.0193251371383667, + "1881223807/train1": 0.0028591901063919067, + "1881223807/train2": 0.011455714702606201, + "1919198188/train0": 0.05272909998893738, + "1919198188/train1": 0.009045466780662537, + "1919198188/train2": 0.025552168488502502, + "1991414747/train0": 0.0, + "1991414747/train1": 0.0, + "1991414747/train2": 0.0, + "1997090716/train0": 1.8607252091169357, + "1997090716/train1": 1.8607252091169357, + "1997090716/train2": 1.8607252091169357, + "2015956957/train0": 0.38412333317101, + "2015956957/train1": -0.010827693343162536, + "2015956957/train2": 0.3726910464465618, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": -0.02582019567489624, + "211245842/train1": -0.0029901862144470215, + "211245842/train2": 1.1611655950546265, + "2141802816/train0": 0.05392231047153473, + "2141802816/train1": 0.02508598566055298, + "2141802816/train2": 1.9291927516460419, + "256770874/train0": -0.6146733313798904, + "256770874/train1": -0.04324805736541748, + "256770874/train2": -0.6451330631971359, + "278008524/train0": -0.09603888296987861, + "278008524/train1": -0.07600005972199142, + "278008524/train2": -0.0746062200050801, + "416183106/train0": 0.37748780846595764, + "416183106/train1": 1.7996735870838165, + "416183106/train2": 1.7996735870838165, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0011172294616699219, + "551504635/train1": -0.00224381685256958, + "551504635/train2": 1.276113212108612, + "565369392/train0": -0.014494215448697409, + "565369392/train1": -0.6261008381843567, + "565369392/train2": -0.6670952091614405, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": -0.019304243847727776, + "585904130/train1": -0.04237352559963862, + "585904130/train2": 0.0031885181864102683, + "626486813/train0": 0.0560848613968119, + "626486813/train1": -0.43178365693893284, + "626486813/train2": -0.43540495017077774, + "636680137/train0": -1.9875853462144732, + "636680137/train1": -1.986071527004242, + "636680137/train2": -1.9756397809833288, + "706479600/train0": 1.1874593496322632, + "706479600/train1": 1.1649112105369568, + "706479600/train2": 1.1649112105369568, + "750448296/train0": -0.004540383815765381, + "750448296/train1": -0.03810691833496094, + "750448296/train2": -0.044590720906853676, + "75070641/train0": 0.03692302107810974, + "75070641/train1": 0.6101358532905579, + "75070641/train2": 0.36233198642730713, + "767747424/train0": 0.022872742265462875, + "767747424/train1": -0.4340953875798732, + "767747424/train2": -0.4351339202839881, + "77147264/train0": 0.5784205198287964, + "77147264/train1": 0.005056649446487427, + "77147264/train2": 0.5784205198287964, + "794181626/train0": 1.826662853360176, + "794181626/train1": 1.8190439492464066, + "794181626/train2": 1.8190439492464066, + "957101489/train0": 0.3842049241065979, + "957101489/train1": 1.7962781339883804, + "957101489/train2": 1.7960350513458252, + "998761162/train0": 0.015134349465370178, + "998761162/train1": 1.9321834295988083, + "998761162/train2": 1.9321834295988083 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5660580151865624, + "low": -0.01856180906115269, + "mean": 0.28212140274398645, + "n": 5, + "num_samples": 500 + }, + "mean": 0.28212140274398645, + "per_task": { + "LiftPegUpright-v1": 0.2717691792382134, + "PickCube-v1": 0.2120050440231959, + "PullCube-v1": -0.21874524904905782, + "PushCube-v1": 0.7821719508904678, + "StackCube-v1": 0.3634060886171129 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2666971658560074, + "low": 0.03155638067167621, + "mean": 0.14528417887851136, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3537825059101655, + "low": 0.21040189125295505, + "mean": 0.284160756501182, + "n": 141, + "num_samples": 500 + }, + "mean": 0.284160756501182, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 1.0, + "1176943004/train1": 0.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 1.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 1.0, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 0.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 0.0, + "1607773860/train1": 0.0, + "1607773860/train2": 1.0, + "1665739511/train0": 0.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 1.0, + "1991414747/train1": 0.0, + "1991414747/train2": 1.0, + "1997090716/train0": 0.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 0.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 1.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 0.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.0, + "565369392/train2": 0.0, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "706479600/train0": 0.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 0.0, + "77147264/train1": 0.5, + "77147264/train2": 1.0, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 0.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.39418803418803416, + "low": 0.07777777777777777, + "mean": 0.22102564102564098, + "n": 5, + "num_samples": 500 + }, + "mean": 0.22102564102564098, + "per_task": { + "LiftPegUpright-v1": 0.2222222222222222, + "PickCube-v1": 0.0, + "PullCube-v1": 0.16666666666666666, + "PushCube-v1": 0.5384615384615384, + "StackCube-v1": 0.17777777777777778 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.30434782608695654, + "low": 0.18840579710144928, + "mean": 0.24154589371980675, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.05035460992907801, + "low": -0.13687943262411348, + "mean": -0.038534278959810875, + "n": 141, + "num_samples": 500 + }, + "mean": -0.038534278959810875, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": -1.0, + "1055567618/train1": -1.0, + "1055567618/train2": -1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.0, + "1176943004/train1": 0.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.5, + "1271718007/train1": 0.0, + "1271718007/train2": 0.0, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 0.0, + "1375492786/train0": -1.0, + "1375492786/train1": -1.0, + "1375492786/train2": -1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": -1.0, + "1475440973/train2": -1.0, + "1483645336/train0": 1.0, + "1483645336/train1": -1.0, + "1483645336/train2": -1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": -1.0, + "1549324805/train0": 0.0, + "1549324805/train1": -0.3333333333333333, + "1549324805/train2": -0.3333333333333333, + "1607773860/train0": -1.0, + "1607773860/train1": -1.0, + "1607773860/train2": 0.0, + "1665739511/train0": -1.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 0.0, + "1991414747/train1": -1.0, + "1991414747/train2": 0.0, + "1997090716/train0": 0.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 0.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": -1.0, + "534717141/train1": -1.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 0.0, + "565369392/train0": 0.0, + "565369392/train1": -0.3333333333333333, + "565369392/train2": -0.3333333333333333, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": -0.25, + "626486813/train2": -0.25, + "636680137/train0": -1.0, + "636680137/train1": -1.0, + "636680137/train2": -1.0, + "706479600/train0": 0.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": -0.25, + "767747424/train2": -0.25, + "77147264/train0": -0.5, + "77147264/train1": 0.0, + "77147264/train2": 0.5, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 0.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.08102564102564103, + "low": -0.09986928104575163, + "mean": -0.01625942684766214, + "n": 5, + "num_samples": 500 + }, + "mean": -0.01625942684766214, + "per_task": { + "LiftPegUpright-v1": -0.1111111111111111, + "PickCube-v1": 0.0, + "PullCube-v1": -0.12745098039215685, + "PushCube-v1": 0.1794871794871795, + "StackCube-v1": -0.022222222222222223 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.024154589371980676, + "low": -0.1111111111111111, + "mean": -0.043478260869565216, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8244129490417189, + "low": 0.5588968922524719, + "mean": 0.6978370489784622, + "n": 141, + "num_samples": 500 + }, + "mean": 0.6978370489784622, + "per_seed": { + "1025740212/train0": 0.21424125134944916, + "1025740212/train1": 0.2209487408399582, + "1025740212/train2": 0.22749333083629608, + "1055157354/train0": 0.14835423231124878, + "1055157354/train1": 0.18621200323104858, + "1055157354/train2": 0.18986818194389343, + "1055567618/train0": 0.8168438673019409, + "1055567618/train1": 0.17464876174926758, + "1055567618/train2": 0.07813572883605957, + "1113268940/train0": 0.27990618348121643, + "1113268940/train1": 0.47868117690086365, + "1113268940/train2": 0.6364428400993347, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 2.0, + "1176943004/train1": 0.02513018250465393, + "1176943004/train2": 2.0, + "1189392917/train0": 2.0, + "1189392917/train1": 2.0, + "1189392917/train2": 2.0, + "1231780997/train0": 0.22899846732616425, + "1231780997/train1": 2.0, + "1231780997/train2": 0.22716176509857178, + "1271718007/train0": 2.0, + "1271718007/train1": 1.10751623660326, + "1271718007/train2": 1.0662197396159172, + "1273247072/train0": 0.13278456032276154, + "1273247072/train1": 0.04248468950390816, + "1273247072/train2": 0.10470707714557648, + "1358421325/train0": 0.23671336472034454, + "1358421325/train1": 0.1808367669582367, + "1358421325/train2": 0.09903085231781006, + "1375492786/train0": 0.001358628273010254, + "1375492786/train1": 0.001317779184319079, + "1375492786/train2": 0.0005992253427393734, + "141984615/train0": 0.04657847310105959, + "141984615/train1": 0.03509971871972084, + "141984615/train2": 0.04040361692508062, + "1475440973/train0": 2.0, + "1475440973/train1": 0.710078775882721, + "1475440973/train2": 0.7161068916320801, + "1483645336/train0": 2.0, + "1483645336/train1": 0.06921124458312988, + "1483645336/train2": 0.0313941091299057, + "1522272686/train0": 0.2340310961008072, + "1522272686/train1": 0.19632326066493988, + "1522272686/train2": 0.17692717909812927, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 0.7195272445678711, + "1549324805/train0": 1.3393354682872693, + "1549324805/train1": 0.6769271823577583, + "1549324805/train2": 0.6709667841593424, + "1607773860/train0": 0.220267653465271, + "1607773860/train1": 0.2231033444404602, + "1607773860/train2": 2.0, + "1665739511/train0": 0.8140708208084106, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 0.08418524265289307, + "1747178030/train2": 0.09071242809295654, + "1881223807/train0": 0.03111603856086731, + "1881223807/train1": 0.025857940316200256, + "1881223807/train2": 0.018627971410751343, + "1919198188/train0": 0.07307347655296326, + "1919198188/train1": 0.046244338154792786, + "1919198188/train2": 0.043911516666412354, + "1991414747/train0": 2.0, + "1991414747/train1": 0.013612315058708191, + "1991414747/train2": 2.0, + "1997090716/train0": 0.03083018958568573, + "1997090716/train1": 2.0, + "1997090716/train2": 2.0, + "2015956957/train0": 0.433539473824203, + "2015956957/train1": 0.034096240811049935, + "2015956957/train2": 0.4342371065169573, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 0.22217948734760284, + "211245842/train1": 0.15655378997325897, + "211245842/train2": 2.0, + "2141802816/train0": 0.0492904931306839, + "2141802816/train1": 0.07612672448158264, + "2141802816/train2": 0.13836362957954407, + "256770874/train0": 0.13286492228507996, + "256770874/train1": 0.10803383588790894, + "256770874/train2": 0.16439804434776306, + "278008524/train0": 0.009787360846530646, + "278008524/train1": 0.020062235475052148, + "278008524/train2": 0.012664265930652618, + "416183106/train0": 0.24176457524299622, + "416183106/train1": 0.23312881588935852, + "416183106/train2": 0.18704846501350403, + "534717141/train0": 0.07250408083200455, + "534717141/train1": 0.04410672187805176, + "534717141/train2": 2.0, + "551504635/train0": 0.6918362975120544, + "551504635/train1": 0.7224790453910828, + "551504635/train2": 0.726307213306427, + "565369392/train0": 0.7208344464500746, + "565369392/train1": 0.12740397453308105, + "565369392/train2": 0.08709763859709103, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.012818952556699514, + "585904130/train1": 0.02085024770349264, + "585904130/train2": 0.06744805444031954, + "626486813/train0": 0.5429544361541048, + "626486813/train1": 0.04864858591463417, + "626486813/train2": 0.03831234632525593, + "636680137/train0": 0.012414653785526752, + "636680137/train1": 0.013928472995758057, + "636680137/train2": 0.017868518829345703, + "706479600/train0": 0.8134256601333618, + "706479600/train1": 2.0, + "706479600/train2": 2.0, + "750448296/train0": 0.029670913703739643, + "750448296/train1": 0.046416282653808594, + "750448296/train2": 0.049234201200306416, + "75070641/train0": 0.16485285758972168, + "75070641/train1": 0.22657974064350128, + "75070641/train2": 0.194048672914505, + "767747424/train0": 0.554436425678432, + "767747424/train1": 0.06781683932058513, + "767747424/train2": 0.0877318112179637, + "77147264/train0": 0.13140545785427094, + "77147264/train1": 1.426636129617691, + "77147264/train2": 2.0, + "794181626/train0": 2.0, + "794181626/train1": 2.0, + "794181626/train2": 2.0, + "957101489/train0": 0.5881698727607727, + "957101489/train1": 0.22424602508544922, + "957101489/train2": 0.567138671875, + "998761162/train0": 0.006543517112731934, + "998761162/train1": 0.04874563217163086, + "998761162/train2": 0.0608120858669281 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.886751612873159, + "low": 0.2852983943786886, + "mean": 0.5914274646835725, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5914274646835725, + "per_task": { + "LiftPegUpright-v1": 0.8222338457902273, + "PickCube-v1": 0.09332544232408206, + "PullCube-v1": 0.372572739694756, + "PushCube-v1": 1.0957474731481993, + "StackCube-v1": 0.5732578224605984 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.7023529558462779, + "low": 0.48535686017352647, + "mean": 0.5875041471669958, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.01412016182106394, + "low": -0.30047688159442687, + "mean": -0.14455097152575763, + "n": 141, + "num_samples": 500 + }, + "mean": -0.14455097152575763, + "per_seed": { + "1025740212/train0": -0.3141324073076248, + "1025740212/train1": -0.3074249178171158, + "1025740212/train2": -0.3008803278207779, + "1055157354/train0": -0.02813686430454254, + "1055157354/train1": 0.009720906615257263, + "1055157354/train2": -0.006615892052650452, + "1055567618/train0": -1.183156132698059, + "1055567618/train1": -1.8253512382507324, + "1055567618/train2": -1.9218642711639404, + "1113268940/train0": -0.14240634441375732, + "1113268940/train1": 0.05636864900588989, + "1113268940/train2": 0.21413031220436096, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.949059471487999, + "1176943004/train1": -0.032903000712394714, + "1176943004/train2": 1.949059471487999, + "1189392917/train0": 1.8915813565254211, + "1189392917/train1": 1.8915813565254211, + "1189392917/train2": 1.8915813565254211, + "1231780997/train0": -0.498007670044899, + "1231780997/train1": 1.2729938626289368, + "1231780997/train2": -0.49984437227249146, + "1271718007/train0": 0.5857395231723785, + "1271718007/train1": -0.3078648969531059, + "1271718007/train2": -0.34916139394044876, + "1273247072/train0": -0.02064509689807892, + "1273247072/train1": -0.1109449677169323, + "1273247072/train2": -0.04872258007526398, + "1358421325/train0": -0.5959889143705368, + "1358421325/train1": -0.6516397297382355, + "1358421325/train2": -0.7336714267730713, + "1375492786/train0": -1.9986413717269897, + "1375492786/train1": -1.998682220815681, + "1375492786/train2": -1.9994007746572606, + "141984615/train0": -0.04723366598288218, + "141984615/train1": -0.05871242036422094, + "141984615/train2": -0.05340852215886116, + "1475440973/train0": 0.0, + "1475440973/train1": -1.289921224117279, + "1475440973/train2": -1.28389310836792, + "1483645336/train0": 1.9835839420557022, + "1483645336/train1": -1.9307887554168701, + "1483645336/train2": -1.9686058908700943, + "1522272686/train0": 0.03926999866962433, + "1522272686/train1": -0.0021198540925979614, + "1522272686/train2": -0.01783391833305359, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": -1.280472755432129, + "1549324805/train0": -0.03941198003788789, + "1549324805/train1": -0.7018202659673989, + "1549324805/train2": -0.7077806641658148, + "1607773860/train0": -1.779732346534729, + "1607773860/train1": -1.7768966555595398, + "1607773860/train2": 0.0, + "1665739511/train0": -1.1859291791915894, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.8536934107542038, + "1747178030/train1": -0.06212134659290314, + "1747178030/train2": -0.05559416115283966, + "1881223807/train0": -0.0009109526872634888, + "1881223807/train1": -0.006169050931930542, + "1881223807/train2": -0.013399019837379456, + "1919198188/train0": 0.0358746200799942, + "1919198188/train1": 0.009045466780662537, + "1919198188/train2": 0.0067126452922821045, + "1991414747/train0": 0.0, + "1991414747/train1": -1.9863876849412918, + "1991414747/train2": 0.0, + "1997090716/train0": -0.10844460129737854, + "1997090716/train1": 1.8607252091169357, + "1997090716/train2": 1.8607252091169357, + "2015956957/train0": 0.3664196839556098, + "2015956957/train1": -0.033023549057543275, + "2015956957/train2": 0.36711731664836406, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": -0.6166549175977707, + "211245842/train1": -0.6822806149721146, + "211245842/train2": 1.1611655950546265, + "2141802816/train0": -0.02151675522327423, + "2141802816/train1": 0.005319476127624512, + "2141802816/train2": 0.06755638122558594, + "256770874/train0": -0.7228831350803375, + "256770874/train1": -0.7477142214775085, + "256770874/train2": -0.6913500130176544, + "278008524/train0": -0.09768450754927471, + "278008524/train1": -0.08740963292075321, + "278008524/train2": -0.09480760246515274, + "416183106/train0": 0.041438162326812744, + "416183106/train1": 0.03280240297317505, + "416183106/train2": -0.013277947902679443, + "534717141/train0": -1.9274959191679955, + "534717141/train1": -1.9558932781219482, + "534717141/train2": 0.0, + "551504635/train0": -0.03213626146316528, + "551504635/train1": -0.00224381685256958, + "551504635/train2": 0.0024204254150390625, + "565369392/train0": -0.07893120000759761, + "565369392/train1": -0.6723616719245911, + "565369392/train2": -0.712668007860581, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": -0.06247314546878139, + "585904130/train1": -0.05444185032198826, + "585904130/train2": -0.007844043585161367, + "626486813/train0": 0.037514419527724385, + "626486813/train1": -0.4567914307117462, + "626486813/train2": -0.46712767030112445, + "636680137/train0": -1.9875853462144732, + "636680137/train1": -1.986071527004242, + "636680137/train2": -1.9821314811706543, + "706479600/train0": 0.000885009765625, + "706479600/train1": 1.1649112105369568, + "706479600/train2": 1.1649112105369568, + "750448296/train0": -0.08186264615505934, + "750448296/train1": -0.06511727720499039, + "750448296/train2": -0.062299358658492565, + "75070641/train0": -0.03789877891540527, + "75070641/train1": 0.02382810413837433, + "75070641/train2": -0.008702963590621948, + "767747424/train0": 0.006912237964570522, + "767747424/train1": -0.47970734839327633, + "767747424/train2": -0.45979237649589777, + "77147264/train0": -1.2901740223169327, + "77147264/train1": 0.005056649446487427, + "77147264/train2": 0.5784205198287964, + "794181626/train0": 1.826662853360176, + "794181626/train1": 1.8190439492464066, + "794181626/train2": 1.8190439492464066, + "957101489/train0": 0.3842049241065979, + "957101489/train1": 0.02052415907382965, + "957101489/train2": 0.3631737232208252, + "998761162/train0": -0.06236153841018677, + "998761162/train1": -0.019070938229560852, + "998761162/train2": -0.007004484534263611 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.09991136278861609, + "low": -0.2553936764363997, + "mean": -0.09776939829294695, + "n": 5, + "num_samples": 500 + }, + "mean": -0.09776939829294695, + "per_task": { + "LiftPegUpright-v1": -0.24474721981419456, + "PickCube-v1": -0.0601042148967584, + "PullCube-v1": -0.2786571810885981, + "PushCube-v1": 0.3248212047112294, + "StackCube-v1": -0.23015958037641313 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": -0.02365911047419776, + "low": -0.26539749033037113, + "mean": -0.14829927152455763, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5224681463797662, + "low": 0.2913920145155972, + "mean": 0.40561739670197494, + "n": 141, + "num_samples": 500 + }, + "mean": 0.40561739670197494, + "per_seed": { + "1025740212/train0": 0.26188503205776215, + "1025740212/train1": 0.47739143669605255, + "1025740212/train2": 1.772506669163704, + "1055157354/train0": 1.8516457676887512, + "1055157354/train1": 0.6370042562484741, + "1055157354/train2": 0.03480546176433563, + "1055567618/train0": 1.183156132698059, + "1055567618/train1": 1.8253512382507324, + "1055567618/train2": 1.9218642711639404, + "1113268940/train0": 0.38470593094825745, + "1113268940/train1": 0.18306860327720642, + "1113268940/train2": 0.06608206033706665, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.974869817495346, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 0.4897426515817642, + "1231780997/train1": 0.0, + "1231780997/train2": 0.5006794333457947, + "1271718007/train0": 0.0, + "1271718007/train1": 0.00353144109249115, + "1271718007/train2": 0.19734752923250198, + "1273247072/train0": 0.0, + "1273247072/train1": 0.45726464316248894, + "1273247072/train2": 0.35906313359737396, + "1358421325/train0": 0.5775325745344162, + "1358421325/train1": 0.6467066705226898, + "1358421325/train2": 1.90096914768219, + "1375492786/train0": 0.0002854267368093133, + "1375492786/train1": 0.00047226750757545233, "1375492786/train2": 0.0002880295505747199, "141984615/train0": 0.03316789741317431, "141984615/train1": 0.05730179324746132, "141984615/train2": 0.013830550014972687, "1475440973/train0": 0.0, - "1475440973/train1": 1.289921224117279, - "1475440973/train2": 1.28389310836792, + "1475440973/train1": 1.289921224117279, + "1475440973/train2": 1.28389310836792, + "1483645336/train0": 0.0, + "1483645336/train1": 1.9307887554168701, + "1483645336/train2": 1.9686058908700943, + "1522272686/train0": 0.009524375200271606, + "1522272686/train1": 0.03616611659526825, + "1522272686/train2": 0.017972871661186218, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 1.280472755432129, + "1549324805/train0": 0.0034310482442378998, + "1549324805/train1": 0.0015795694974561532, + "1549324805/train2": 0.01237945475926002, + "1607773860/train0": 1.779732346534729, + "1607773860/train1": 0.6036040186882019, + "1607773860/train2": 0.0, + "1665739511/train0": 0.007551252841949463, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.027157030999660492, + "1747178030/train2": 0.034092940390110016, + "1881223807/train0": 0.020236089825630188, + "1881223807/train1": 0.009028241038322449, + "1881223807/train2": 0.024854734539985657, + "1919198188/train0": 0.016854479908943176, + "1919198188/train1": 0.0, + "1919198188/train2": 0.018839523196220398, + "1991414747/train0": 0.0, + "1991414747/train1": 1.9863876849412918, + "1991414747/train2": 0.0, + "1997090716/train0": 1.9691698104143143, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.01770364921540022, + "2015956957/train1": 0.022195855714380743, + "2015956957/train2": 0.005573729798197746, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.5908347219228745, + "211245842/train1": 0.6792904287576675, + "211245842/train2": 0.0, + "2141802816/train0": 0.07543906569480896, + "2141802816/train1": 0.019766509532928467, + "2141802816/train2": 1.861636370420456, + "256770874/train0": 0.10820980370044708, + "256770874/train1": 0.7044661641120911, + "256770874/train2": 0.046216949820518494, + "278008524/train0": 0.0016456245793960989, + "278008524/train1": 0.011409573198761791, + "278008524/train2": 0.020201382460072637, + "416183106/train0": 0.3360496461391449, + "416183106/train1": 1.7668711841106415, + "416183106/train2": 1.812951534986496, + "534717141/train0": 1.9274959191679955, + "534717141/train1": 1.9558932781219482, + "534717141/train2": 0.0, + "551504635/train0": 0.033253490924835205, + "551504635/train1": 0.0, + "551504635/train2": 1.273692786693573, + "565369392/train0": 0.06443698455890019, + "565369392/train1": 0.046260833740234375, + "565369392/train2": 0.04557279869914055, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.043168901621053614, + "585904130/train1": 0.012068324722349644, + "585904130/train2": 0.011032561771571636, + "626486813/train0": 0.018570441869087517, + "626486813/train1": 0.02500777377281338, + "626486813/train2": 0.031722720130346715, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.006491700187325478, + "706479600/train0": 1.1865743398666382, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.07732226233929396, + "750448296/train1": 0.02701035887002945, + "750448296/train2": 0.01770863775163889, + "75070641/train0": 0.07482179999351501, + "75070641/train1": 0.5863077491521835, + "75070641/train2": 0.3710349500179291, + "767747424/train0": 0.015960504300892353, + "767747424/train1": 0.04561196081340313, + "767747424/train2": 0.024658456211909652, + "77147264/train0": 1.868594542145729, + "77147264/train1": 0.0, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 0.0, + "957101489/train1": 1.7757539749145508, + "957101489/train2": 1.432861328125, + "998761162/train0": 0.07749588787555695, + "998761162/train1": 1.9512543678283691, + "998761162/train2": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5390796998678109, + "low": 0.18723032816778867, + "mean": 0.3798908010369334, + "n": 5, + "num_samples": 500 + }, + "mean": 0.3798908010369334, + "per_task": { + "LiftPegUpright-v1": 0.516516399052408, + "PickCube-v1": 0.2721092589199543, + "PullCube-v1": 0.05991193203954026, + "PushCube-v1": 0.4573507461792383, + "StackCube-v1": 0.593565668993526 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.37381450664971905, + "low": 0.2099735496069669, + "mean": 0.293583450403069, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2624113475177305, + "low": 0.1276595744680851, + "mean": 0.19148936170212766, + "n": 141, + "num_samples": 500 + }, + "mean": 0.19148936170212766, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 0.0, + "1231780997/train1": 0.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.0, + "1271718007/train1": 0.0, + "1271718007/train2": 0.0, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 0.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1607773860/train0": 1.0, + "1607773860/train1": 0.0, + "1607773860/train2": 0.0, + "1665739511/train0": 0.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 0.0, + "1991414747/train1": 1.0, + "1991414747/train2": 0.0, + "1997090716/train0": 1.0, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.0, + "2015956957/train1": 0.0, + "2015956957/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 1.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 1.0, + "565369392/train0": 0.0, + "565369392/train1": 0.0, + "565369392/train2": 0.0, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "706479600/train0": 1.0, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 0.0, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 0.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 0.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2594871794871795, + "low": 0.05725490196078431, + "mean": 0.15896430367018605, + "n": 5, + "num_samples": 500 + }, + "mean": 0.15896430367018605, + "per_task": { + "LiftPegUpright-v1": 0.2777777777777778, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0196078431372549, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.26666666666666666 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.18840579710144928, + "low": 0.08695652173913043, + "mean": 0.13526570048309178, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.43522458628841615, + "low": 0.2907801418439716, + "mean": 0.3619385342789597, + "n": 141, + "num_samples": 500 + }, + "mean": 0.3619385342789597, + "per_seed": { + "1025740212/train0": 1.0, + "1025740212/train1": 1.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 1.0, + "1055157354/train2": 1.0, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 1.0, + "1113268940/train1": 1.0, + "1113268940/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 0.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 1.0, + "1231780997/train1": 0.0, + "1231780997/train2": 1.0, + "1271718007/train0": 0.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 1.0, + "1273247072/train1": 1.0, + "1273247072/train2": 1.0, + "1358421325/train0": 1.0, + "1358421325/train1": 1.0, + "1358421325/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 0.0, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 0.0, + "1607773860/train1": 1.0, + "1607773860/train2": 0.0, + "1665739511/train0": 1.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1881223807/train0": 1.0, + "1881223807/train1": 1.0, + "1881223807/train2": 1.0, + "1919198188/train0": 1.0, + "1919198188/train1": 1.0, + "1919198188/train2": 1.0, + "1991414747/train0": 0.0, + "1991414747/train1": 0.0, + "1991414747/train2": 0.0, + "1997090716/train0": 0.0, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.0, + "2015956957/train1": 0.2, + "2015956957/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 0.0, + "2141802816/train0": 1.0, + "2141802816/train1": 1.0, + "2141802816/train2": 0.0, + "256770874/train0": 1.0, + "256770874/train1": 1.0, + "256770874/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "416183106/train0": 1.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 0.0, + "551504635/train0": 1.0, + "551504635/train1": 1.0, + "551504635/train2": 0.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.6666666666666666, + "565369392/train2": 0.6666666666666666, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 0.0, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 1.0, + "75070641/train1": 1.0, + "75070641/train2": 1.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 0.0, + "77147264/train1": 0.5, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 1.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 1.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7572649572649572, + "low": 0.24550025138260434, + "mean": 0.48667672197083967, + "n": 5, + "num_samples": 500 + }, + "mean": 0.48667672197083967, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 1.0, + "PullCube-v1": 0.14705882352941177, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.5555555555555556 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.36231884057971014, + "low": 0.2318840579710145, + "mean": 0.2946859903381642, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6320330969267138, + "low": 0.47304964539007094, + "mean": 0.5534278959810874, + "n": 141, + "num_samples": 500 + }, + "mean": 0.5534278959810874, + "per_seed": { + "1025740212/train0": 1.0, + "1025740212/train1": 1.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 1.0, + "1055157354/train2": 1.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 1.0, + "1113268940/train1": 1.0, + "1113268940/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 1.0, + "1231780997/train1": 0.0, + "1231780997/train2": 1.0, + "1271718007/train0": 0.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 1.0, + "1273247072/train1": 1.0, + "1273247072/train2": 1.0, + "1358421325/train0": 1.0, + "1358421325/train1": 1.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, "1483645336/train0": 0.0, - "1483645336/train1": 1.9307887554168701, - "1483645336/train2": 1.9686058908700943, - "1522272686/train0": 0.009524375200271606, - "1522272686/train1": 0.03616611659526825, - "1522272686/train2": 0.017972871661186218, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, "1528740417/train0": 0.0, "1528740417/train1": 0.0, - "1528740417/train2": 1.280472755432129, - "1549324805/train0": 0.0034310482442378998, - "1549324805/train1": 0.0015795694974561532, - "1549324805/train2": 0.01237945475926002, - "1607773860/train0": 1.779732346534729, - "1607773860/train1": 0.6036040186882019, + "1528740417/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 1.0, + "1607773860/train1": 1.0, "1607773860/train2": 0.0, - "1665739511/train0": 0.007551252841949463, + "1665739511/train0": 1.0, "1665739511/train1": 0.0, "1665739511/train2": 0.0, "1747178030/train0": 0.0, - "1747178030/train1": 0.027157030999660492, - "1747178030/train2": 0.034092940390110016, - "1881223807/train0": 0.020236089825630188, - "1881223807/train1": 0.009028241038322449, - "1881223807/train2": 0.024854734539985657, - "1919198188/train0": 0.016854479908943176, - "1919198188/train1": 0.0, - "1919198188/train2": 0.018839523196220398, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1881223807/train0": 1.0, + "1881223807/train1": 1.0, + "1881223807/train2": 1.0, + "1919198188/train0": 1.0, + "1919198188/train1": 1.0, + "1919198188/train2": 1.0, "1991414747/train0": 0.0, - "1991414747/train1": 1.9863876849412918, + "1991414747/train1": 1.0, "1991414747/train2": 0.0, - "1997090716/train0": 1.9691698104143143, + "1997090716/train0": 1.0, "1997090716/train1": 0.0, "1997090716/train2": 0.0, - "2015956957/train0": 0.01770364921540022, - "2015956957/train1": 0.022195855714380743, - "2015956957/train2": 0.005573729798197746, + "2015956957/train0": 0.0, + "2015956957/train1": 0.2, + "2015956957/train2": 0.0, "2054899126/train0": 0.0, "2054899126/train1": 0.0, "2054899126/train2": 0.0, - "211245842/train0": 0.5908347219228745, - "211245842/train1": 0.6792904287576675, + "211245842/train0": 1.0, + "211245842/train1": 1.0, "211245842/train2": 0.0, - "2141802816/train0": 0.07543906569480896, - "2141802816/train1": 0.019766509532928467, - "2141802816/train2": 1.861636370420456, - "256770874/train0": 0.10820980370044708, - "256770874/train1": 0.7044661641120911, - "256770874/train2": 0.046216949820518494, - "278008524/train0": 0.0016456245793960989, - "278008524/train1": 0.011409573198761791, - "278008524/train2": 0.020201382460072637, - "416183106/train0": 0.3360496461391449, - "416183106/train1": 1.7668711841106415, - "416183106/train2": 1.812951534986496, - "534717141/train0": 1.9274959191679955, - "534717141/train1": 1.9558932781219482, + "2141802816/train0": 1.0, + "2141802816/train1": 1.0, + "2141802816/train2": 1.0, + "256770874/train0": 1.0, + "256770874/train1": 1.0, + "256770874/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "416183106/train0": 1.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, "534717141/train2": 0.0, - "551504635/train0": 0.033253490924835205, - "551504635/train1": 0.0, - "551504635/train2": 1.273692786693573, - "565369392/train0": 0.06443698455890019, - "565369392/train1": 0.046260833740234375, - "565369392/train2": 0.04557279869914055, + "551504635/train0": 1.0, + "551504635/train1": 1.0, + "551504635/train2": 1.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.6666666666666666, + "565369392/train2": 0.6666666666666666, "565621826/train0": 0.0, "565621826/train1": 0.0, "565621826/train2": 0.0, - "585904130/train0": 0.043168901621053614, - "585904130/train1": 0.012068324722349644, - "585904130/train2": 0.011032561771571636, - "626486813/train0": 0.018570441869087517, - "626486813/train1": 0.02500777377281338, - "626486813/train2": 0.031722720130346715, - "636680137/train0": 0.0, - "636680137/train1": 0.0, - "636680137/train2": 0.006491700187325478, - "706479600/train0": 1.1865743398666382, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 1.0, "706479600/train1": 0.0, "706479600/train2": 0.0, - "750448296/train0": 0.07732226233929396, - "750448296/train1": 0.02701035887002945, - "750448296/train2": 0.01770863775163889, - "75070641/train0": 0.07482179999351501, - "75070641/train1": 0.5863077491521835, - "75070641/train2": 0.3710349500179291, - "767747424/train0": 0.015960504300892353, - "767747424/train1": 0.04561196081340313, - "767747424/train2": 0.024658456211909652, - "77147264/train0": 1.868594542145729, - "77147264/train1": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 1.0, + "75070641/train1": 1.0, + "75070641/train2": 1.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 0.5, "77147264/train2": 0.0, "794181626/train0": 0.0, "794181626/train1": 0.0, "794181626/train2": 0.0, - "957101489/train0": 0.0, - "957101489/train1": 1.7757539749145508, - "957101489/train2": 1.432861328125, - "998761162/train0": 0.07749588787555695, - "998761162/train1": 1.9512543678283691, - "998761162/train2": 1.939187914133072 + "957101489/train0": 1.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 1.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 } }, "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.5319262530406317, - "low": 0.19367229081819662, - "mean": 0.3798908010369334, + "high": 0.8844444444444444, + "low": 0.3478632478632479, + "mean": 0.6456410256410257, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, - "mean": 0.3798908010369334, + "mean": 0.6456410256410257, "per_task": { - "LiftPegUpright-v1": 0.516516399052408, - "PickCube-v1": 0.2721092589199543, - "PullCube-v1": 0.05991193203954026, - "PushCube-v1": 0.4573507461792383, - "StackCube-v1": 0.593565668993526 + "LiftPegUpright-v1": 0.7777777777777778, + "PickCube-v1": 1.0, + "PullCube-v1": 0.16666666666666666, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.8222222222222222 } }, - "metric": "selector_regret_at_8", + "metric": "success_total_car_to_hidden_at_8", "micro": { "confidence": 0.95, - "high": 0.3721372773162211, - "low": 0.21313571063115527, - "mean": 0.293583450403069, + "high": 0.4975845410628019, + "low": 0.357487922705314, + "mean": 0.42995169082125606, "n": 207, - "num_samples": 1000 + "num_samples": 500 }, "n": 207 }, @@ -3023,11 +9095,11 @@ "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.7276497844564805, - "low": 0.49026043312640605, + "high": 0.7238702659522698, + "low": 0.4987892105245752, "mean": 0.6062044484887313, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.6062044484887313, "per_seed": { @@ -3181,7 +9253,7 @@ "low": 0.4585766474838634, "mean": 0.788961660718036, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.788961660718036, "per_task": { @@ -3195,11 +9267,195 @@ "metric": "support_gap_at_8", "micro": { "confidence": 0.95, - "high": 0.6340587634713039, + "high": 0.6411033530127941, "low": 0.4234259423213799, "mean": 0.5282977284379371, "n": 207, - "num_samples": 1000 + "num_samples": 500 + }, + "n": 207 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1419460938234778, + "low": 0.8777518347707955, + "mean": 1.0115917835033712, + "n": 141, + "num_samples": 500 + }, + "mean": 1.0115917835033712, + "per_seed": { + "1025740212/train0": 1.7857587486505508, + "1025740212/train1": 1.7790512591600418, + "1025740212/train2": 1.772506669163704, + "1055157354/train0": 1.8516457676887512, + "1055157354/train1": 1.8137879967689514, + "1055157354/train2": 1.8101318180561066, + "1055567618/train0": 1.183156132698059, + "1055567618/train1": 1.8253512382507324, + "1055567618/train2": 1.9218642711639404, + "1113268940/train0": 1.7200938165187836, + "1113268940/train1": 1.5213188230991364, + "1113268940/train2": 1.3635571599006653, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.974869817495346, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 1.7710015326738358, + "1231780997/train1": 0.0, + "1231780997/train2": 1.7728382349014282, + "1271718007/train0": 0.0, + "1271718007/train1": 0.89248376339674, + "1271718007/train2": 0.9337802603840828, + "1273247072/train0": 1.8672154396772385, + "1273247072/train1": 1.9575153104960918, + "1273247072/train2": 1.8952929228544235, + "1358421325/train0": 1.7632866352796555, + "1358421325/train1": 1.8191632330417633, + "1358421325/train2": 1.90096914768219, + "1375492786/train0": 0.2420923411846161, + "1375492786/train1": 0.24213319027330726, + "1375492786/train2": 0.24285174411488697, + "141984615/train0": 0.15386936937769255, + "141984615/train1": 0.1653481237590313, + "141984615/train2": 0.1600442255536715, + "1475440973/train0": 0.0, + "1475440973/train1": 1.289921224117279, + "1475440973/train2": 1.28389310836792, + "1483645336/train0": 0.0, + "1483645336/train1": 1.9307887554168701, + "1483645336/train2": 1.9686058908700943, + "1522272686/train0": 1.7659689038991928, + "1522272686/train1": 1.8036767393350601, + "1522272686/train2": 1.8230728209018707, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 1.280472755432129, + "1549324805/train0": 0.06341505361100037, + "1549324805/train1": 0.7258233395405114, + "1549324805/train2": 0.7317837377389272, + "1607773860/train0": 1.779732346534729, + "1607773860/train1": 1.7768966555595398, + "1607773860/train2": 0.0, + "1665739511/train0": 1.1859291791915894, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 1.915814757347107, + "1747178030/train2": 1.9092875719070435, + "1881223807/train0": 1.9688839614391327, + "1881223807/train1": 1.9741420596837997, + "1881223807/train2": 1.9813720285892487, + "1919198188/train0": 1.9269265234470367, + "1919198188/train1": 1.9537556618452072, + "1919198188/train2": 1.9560884833335876, + "1991414747/train0": 0.0, + "1991414747/train1": 1.9863876849412918, + "1991414747/train2": 0.0, + "1997090716/train0": 1.9691698104143143, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.09634189326316119, + "2015956957/train1": 0.49578512627631427, + "2015956957/train2": 0.09564426057040691, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 1.7778205126523972, + "211245842/train1": 1.843446210026741, + "211245842/train2": 0.0, + "2141802816/train0": 1.950709506869316, + "2141802816/train1": 1.9238732755184174, + "2141802816/train2": 1.861636370420456, + "256770874/train0": 1.86713507771492, + "256770874/train1": 1.891966164112091, + "256770874/train2": 1.835601955652237, + "278008524/train0": 1.1040238340501674, + "278008524/train1": 1.0937489594216458, + "278008524/train2": 1.1011469289660454, + "416183106/train0": 1.7582354247570038, + "416183106/train1": 1.7668711841106415, + "416183106/train2": 1.812951534986496, + "534717141/train0": 1.9274959191679955, + "534717141/train1": 1.9558932781219482, + "534717141/train2": 0.0, + "551504635/train0": 1.3081637024879456, + "551504635/train1": 1.2775209546089172, + "551504635/train2": 1.273692786693573, + "565369392/train0": 0.7057971333463987, + "565369392/train1": 1.2992276052633922, + "565369392/train2": 1.339533941199382, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.2248280394511918, + "585904130/train1": 0.2167967443043987, + "585904130/train2": 0.1701989375675718, + "626486813/train0": 0.08356922247912735, + "626486813/train1": 0.5638198243686929, + "626486813/train2": 0.5741560639580712, + "636680137/train0": 1.9875853462144732, + "636680137/train1": 1.986071527004242, + "636680137/train2": 1.9821314811706543, + "706479600/train0": 1.1865743398666382, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.20613667648285627, + "750448296/train1": 0.18939130753278732, + "750448296/train2": 0.1865733889862895, + "75070641/train0": 1.8351471424102783, + "75070641/train1": 1.7734202593564987, + "75070641/train2": 1.805951327085495, + "767747424/train0": 0.0750129884108901, + "767747424/train1": 0.12081881728954613, + "767747424/train2": 0.10090384539216757, + "77147264/train0": 1.868594542145729, + "77147264/train1": 0.573363870382309, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 1.4118301272392273, + "957101489/train1": 1.7757539749145508, + "957101489/train2": 1.432861328125, + "998761162/train0": 1.993456482887268, + "998761162/train1": 1.9512543678283691, + "998761162/train2": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6102036754837912, + "low": 0.722751199278172, + "mean": 1.1686045027220224, + "n": 5, + "num_samples": 500 + }, + "mean": 1.1686045027220224, + "per_task": { + "LiftPegUpright-v1": 1.1777661542097728, + "PickCube-v1": 1.9066745576759179, + "PullCube-v1": 0.42758709733321915, + "PushCube-v1": 0.9042525268518008, + "StackCube-v1": 1.4267421775394016 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9329092862770065, + "low": 0.6954710805647476, + "mean": 0.8212702652815715, + "n": 207, + "num_samples": 500 }, "n": 207 } diff --git a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_seed.json b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_seed.json index 7c0c929ddb493554423d370063d7108f4b2ba09b..3d0fc6905b8a769e3be28046bd3e54b589fe86d3 100644 --- a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_seed.json +++ b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_seed.json @@ -1,989 +1,3245 @@ { "1025740212/train0": { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.26188503205776215, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08963387331564551, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4761262834072113, + "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21424125134944916, + "selected_utility_gain_over_base_at_8": -0.3141324073076248, "selector_regret_at_8": 0.26188503205776215, - "support_gap_at_8": 1.5238737165927887 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5238737165927887, + "total_car_to_hidden_at_8": 1.7857587486505508 }, "1025740212/train1": { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 0.47739143669605255, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12992986623548633, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6983401775360107, + "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2209487408399582, + "selected_utility_gain_over_base_at_8": -0.3074249178171158, "selector_regret_at_8": 0.47739143669605255, - "support_gap_at_8": 1.3016598224639893 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3016598224639893, + "total_car_to_hidden_at_8": 1.7790512591600418 }, "1025740212/train2": { + "base_success": 0.0, + "base_utility": 0.528373658657074, "branch_car_at_8": 1.772506669163704, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17388092747958847, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22749333083629608, + "selected_utility_gain_over_base_at_8": -0.3008803278207779, "selector_regret_at_8": 1.772506669163704, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.772506669163704 }, "1055157354/train0": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 1.8516457676887512, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2973966626262834, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14835423231124878, + "selected_utility_gain_over_base_at_8": -0.02813686430454254, "selector_regret_at_8": 1.8516457676887512, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8516457676887512 }, "1055157354/train1": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, "branch_car_at_8": 0.6370042562484741, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07450750352404123, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8232162594795227, + "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18621200323104858, + "selected_utility_gain_over_base_at_8": 0.009720906615257263, "selector_regret_at_8": 0.6370042562484741, - "support_gap_at_8": 1.1767837405204773 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1767837405204773, + "total_car_to_hidden_at_8": 1.8137879967689514 }, "1055157354/train2": { + "base_success": 0.0, + "base_utility": 0.19648407399654388, "branch_car_at_8": 0.03480546176433563, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1805709977888097, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22467364370822906, + "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18986818194389343, + "selected_utility_gain_over_base_at_8": -0.006615892052650452, "selector_regret_at_8": 0.03480546176433563, - "support_gap_at_8": 1.775326356291771 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.775326356291771, + "total_car_to_hidden_at_8": 1.8101318180561066 }, "1055567618/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.183156132698059, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07486610854656817, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8168438673019409, + "selected_utility_gain_over_base_at_8": -1.183156132698059, "selector_regret_at_8": 1.183156132698059, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.183156132698059 }, "1055567618/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.8253512382507324, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.318234334809661, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.17464876174926758, + "selected_utility_gain_over_base_at_8": -1.8253512382507324, "selector_regret_at_8": 1.8253512382507324, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8253512382507324 }, "1055567618/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9218642711639404, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19289744165908876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07813572883605957, + "selected_utility_gain_over_base_at_8": -1.9218642711639404, "selector_regret_at_8": 1.9218642711639404, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9218642711639404 }, "1113268940/train0": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.38470593094825745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1616344352768096, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6646121144294739, + "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.27990618348121643, + "selected_utility_gain_over_base_at_8": -0.14240634441375732, "selector_regret_at_8": 0.38470593094825745, - "support_gap_at_8": 1.3353878855705261 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3353878855705261, + "total_car_to_hidden_at_8": 1.7200938165187836 }, "1113268940/train1": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.18306860327720642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10725283874677279, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6617497801780701, + "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.47868117690086365, + "selected_utility_gain_over_base_at_8": 0.05636864900588989, "selector_regret_at_8": 0.18306860327720642, - "support_gap_at_8": 1.33825021982193 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.33825021982193, + "total_car_to_hidden_at_8": 1.5213188230991364 }, "1113268940/train2": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, "branch_car_at_8": 0.06608206033706665, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.032532089835823584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7025249004364014, + "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6364428400993347, + "selected_utility_gain_over_base_at_8": 0.21413031220436096, "selector_regret_at_8": 0.06608206033706665, - "support_gap_at_8": 1.2974750995635986 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2974750995635986, + "total_car_to_hidden_at_8": 1.3635571599006653 }, "1113371261/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3179396489681421, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1113371261/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.04977883974023356, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1113371261/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.233029795067053, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1176943004/train0": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2176350935010736, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1176943004/train1": { + "base_success": 0.0, + "base_utility": 0.058033183217048645, "branch_car_at_8": 1.974869817495346, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1815411965846294, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02513018250465393, + "selected_utility_gain_over_base_at_8": -0.032903000712394714, "selector_regret_at_8": 1.974869817495346, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.974869817495346 }, "1176943004/train2": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.17468938592004596, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1189392917/train0": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1189392917/train1": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2180235973132697, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1189392917/train2": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1231780997/train0": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.4897426515817642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18322944379343714, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7187411189079285, + "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22899846732616425, + "selected_utility_gain_over_base_at_8": -0.498007670044899, "selector_regret_at_8": 0.4897426515817642, - "support_gap_at_8": 1.2812588810920715 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2812588810920715, + "total_car_to_hidden_at_8": 1.7710015326738358 }, "1231780997/train1": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28322114525972475, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.2729938626289368, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1231780997/train2": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, "branch_car_at_8": 0.5006794333457947, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10207358838016511, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7278411984443665, + "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22716176509857178, + "selected_utility_gain_over_base_at_8": -0.49984437227249146, "selector_regret_at_8": 0.5006794333457947, - "support_gap_at_8": 1.2721588015556335 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2721588015556335, + "total_car_to_hidden_at_8": 1.7728382349014282 }, "1271718007/train0": { + "base_success": 0.5, + "base_utility": 1.4142604768276215, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.07543261691159965, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5857395231723785, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5857395231723785, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1271718007/train1": { + "base_success": 0.5, + "base_utility": 1.415381133556366, "branch_car_at_8": 0.00353144109249115, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07023931523066618, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.1110476776957512, + "proposal_oracle_utility_gain_over_base_at_8": -0.3043334558606148, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.10751623660326, + "selected_utility_gain_over_base_at_8": -0.3078648969531059, "selector_regret_at_8": 0.00353144109249115, - "support_gap_at_8": 0.8889523223042488 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.8889523223042488, + "total_car_to_hidden_at_8": 0.89248376339674 }, "1271718007/train2": { + "base_success": 0.5, + "base_utility": 1.415381133556366, "branch_car_at_8": 0.19734752923250198, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2350719661460603, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.2635672688484192, + "proposal_oracle_utility_gain_over_base_at_8": -0.15181386470794678, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.0662197396159172, + "selected_utility_gain_over_base_at_8": -0.34916139394044876, "selector_regret_at_8": 0.19734752923250198, - "support_gap_at_8": 0.7364327311515808 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.7364327311515808, + "total_car_to_hidden_at_8": 0.9337802603840828 }, "1273247072/train0": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.8672154396772385 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 }, "1273247072/train1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.45726464316248894, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, "selector_regret_at_8": 0.45726464316248894, - "support_gap_at_8": 1.500250667333603 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.500250667333603, + "total_car_to_hidden_at_8": 1.9575153104960918 }, "1273247072/train2": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.35906313359737396, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, "selector_regret_at_8": 0.35906313359737396, - "support_gap_at_8": 1.5362297892570496 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5362297892570496, + "total_car_to_hidden_at_8": 1.8952929228544235 }, "1358421325/train0": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 0.5775325745344162, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13502516765102715, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8142459392547607, + "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23671336472034454, + "selected_utility_gain_over_base_at_8": -0.5959889143705368, "selector_regret_at_8": 0.5775325745344162, - "support_gap_at_8": 1.1857540607452393 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1857540607452393, + "total_car_to_hidden_at_8": 1.7632866352796555 }, "1358421325/train1": { + "base_success": 0.0, + "base_utility": 0.8324764966964722, "branch_car_at_8": 0.6467066705226898, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046694794435454434, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8275434374809265, + "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1808367669582367, + "selected_utility_gain_over_base_at_8": -0.6516397297382355, "selector_regret_at_8": 0.6467066705226898, - "support_gap_at_8": 1.1724565625190735 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1724565625190735, + "total_car_to_hidden_at_8": 1.8191632330417633 }, "1358421325/train2": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, "branch_car_at_8": 1.90096914768219, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.32948159974174274, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09903085231781006, + "selected_utility_gain_over_base_at_8": -0.7336714267730713, "selector_regret_at_8": 1.90096914768219, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.90096914768219 }, "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002854267368093133, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, "selector_regret_at_8": 0.0002854267368093133, - "support_gap_at_8": 0.24180691444780678 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24180691444780678, + "total_car_to_hidden_at_8": 0.2420923411846161 }, "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.00047226750757545233, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14027669602920534, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0017900466918945312, + "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001317779184319079, + "selected_utility_gain_over_base_at_8": -1.998682220815681, "selector_regret_at_8": 0.00047226750757545233, - "support_gap_at_8": 0.2416609227657318 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.2416609227657318, + "total_car_to_hidden_at_8": 0.24213319027330726 }, "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0002880295505747199, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.008822272371479922, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0008872548933140934, + "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0005992253427393734, + "selected_utility_gain_over_base_at_8": -1.9994007746572606, "selector_regret_at_8": 0.0002880295505747199, - "support_gap_at_8": 0.24256371456431225 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24256371456431225, + "total_car_to_hidden_at_8": 0.24285174411488697 }, "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, "branch_car_at_8": 0.03316789741317431, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.10926310602682258, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0797463705142339, + "proposal_oracle_utility_gain_over_base_at_8": -0.01406576856970787, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04657847310105959, + "selected_utility_gain_over_base_at_8": -0.04723366598288218, "selector_regret_at_8": 0.03316789741317431, - "support_gap_at_8": 0.12070147196451823 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12070147196451823, + "total_car_to_hidden_at_8": 0.15386936937769255 }, "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, "branch_car_at_8": 0.05730179324746132, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.08533231968443573, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09240151196718216, + "proposal_oracle_utility_gain_over_base_at_8": -0.001410627116759618, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03509971871972084, + "selected_utility_gain_over_base_at_8": -0.05871242036422094, "selector_regret_at_8": 0.05730179324746132, - "support_gap_at_8": 0.11134201536575954 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.11134201536575954, + "total_car_to_hidden_at_8": 0.1653481237590313 }, "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, "branch_car_at_8": 0.013830550014972687, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.19434281311769708, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.054234166940053306, + "proposal_oracle_utility_gain_over_base_at_8": -0.039577972143888474, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04040361692508062, + "selected_utility_gain_over_base_at_8": -0.05340852215886116, "selector_regret_at_8": 0.013830550014972687, - "support_gap_at_8": 0.14621367553869882 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.14621367553869882, + "total_car_to_hidden_at_8": 0.1600442255536715 }, "1475440973/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.09930428958964055, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1475440973/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.289921224117279, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2443552809335613, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.710078775882721, + "selected_utility_gain_over_base_at_8": -1.289921224117279, "selector_regret_at_8": 1.289921224117279, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.289921224117279 }, "1475440973/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.28389310836792, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22214884767353088, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7161068916320801, + "selected_utility_gain_over_base_at_8": -1.28389310836792, "selector_regret_at_8": 1.28389310836792, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.28389310836792 }, "1483645336/train0": { + "base_success": 0.0, + "base_utility": 0.01641605794429779, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.9835839420557022, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1483645336/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9307887554168701, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13752192684190806, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.06921124458312988, + "selected_utility_gain_over_base_at_8": -1.9307887554168701, "selector_regret_at_8": 1.9307887554168701, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9307887554168701 }, "1483645336/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9686058908700943, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.5995435922806452, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0313941091299057, + "selected_utility_gain_over_base_at_8": -1.9686058908700943, "selector_regret_at_8": 1.9686058908700943, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9686058908700943 }, "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.009524375200271606, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.08257819446483214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2435554713010788, + "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2340310961008072, + "selected_utility_gain_over_base_at_8": 0.03926999866962433, "selector_regret_at_8": 0.009524375200271606, - "support_gap_at_8": 1.7564445286989212 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7564445286989212, + "total_car_to_hidden_at_8": 1.7659689038991928 }, "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.19844311475753784, "branch_car_at_8": 0.03616611659526825, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28555947838938045, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.23248937726020813, + "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.19632326066493988, + "selected_utility_gain_over_base_at_8": -0.0021198540925979614, "selector_regret_at_8": 0.03616611659526825, - "support_gap_at_8": 1.7675106227397919 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7675106227397919, + "total_car_to_hidden_at_8": 1.8036767393350601 }, "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, "branch_car_at_8": 0.017972871661186218, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.3276128524355476, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1949000507593155, + "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.17692717909812927, + "selected_utility_gain_over_base_at_8": -0.01783391833305359, "selector_regret_at_8": 0.017972871661186218, - "support_gap_at_8": 1.8050999492406845 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8050999492406845, + "total_car_to_hidden_at_8": 1.8230728209018707 }, "1528740417/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.015990866806051107, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1528740417/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2757671830981365, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1528740417/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.280472755432129, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16399362968800954, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7195272445678711, + "selected_utility_gain_over_base_at_8": -1.280472755432129, "selector_regret_at_8": 1.280472755432129, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.280472755432129 }, "1549324805/train0": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, "branch_car_at_8": 0.0034310482442378998, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.19382954770507574, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.3427665165315072, + "proposal_oracle_utility_gain_over_base_at_8": -0.035980931793649994, + "selected_success_at_8": 0.6666666666666666, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.3393354682872693, + "selected_utility_gain_over_base_at_8": -0.03941198003788789, "selector_regret_at_8": 0.0034310482442378998, - "support_gap_at_8": 0.05998400536676248 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05998400536676248, + "total_car_to_hidden_at_8": 0.06341505361100037 }, "1549324805/train1": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, "branch_car_at_8": 0.0015795694974561532, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.28625156462411294, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6785067518552145, + "proposal_oracle_utility_gain_over_base_at_8": -0.7002406964699427, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6769271823577583, + "selected_utility_gain_over_base_at_8": -0.7018202659673989, "selector_regret_at_8": 0.0015795694974561532, - "support_gap_at_8": 0.7242437700430552 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7242437700430552, + "total_car_to_hidden_at_8": 0.7258233395405114 }, "1549324805/train2": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, "branch_car_at_8": 0.01237945475926002, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.23864987006465407, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6833462389186025, + "proposal_oracle_utility_gain_over_base_at_8": -0.6954012094065547, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6709667841593424, + "selected_utility_gain_over_base_at_8": -0.7077806641658148, "selector_regret_at_8": 0.01237945475926002, - "support_gap_at_8": 0.7194042829796672 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7194042829796672, + "total_car_to_hidden_at_8": 0.7317837377389272 }, "1607773860/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.779732346534729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08588083120555855, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.220267653465271, + "selected_utility_gain_over_base_at_8": -1.779732346534729, "selector_regret_at_8": 1.779732346534729, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.779732346534729 }, "1607773860/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.6036040186882019, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.18975917078230178, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8267073631286621, + "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.2231033444404602, + "selected_utility_gain_over_base_at_8": -1.7768966555595398, "selector_regret_at_8": 0.6036040186882019, - "support_gap_at_8": 1.173292636871338 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.173292636871338, + "total_car_to_hidden_at_8": 1.7768966555595398 }, "1607773860/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.24177237819615016, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1665739511/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.007551252841949463, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.050940808914706356, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8216220736503601, + "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8140708208084106, + "selected_utility_gain_over_base_at_8": -1.1859291791915894, "selector_regret_at_8": 0.007551252841949463, - "support_gap_at_8": 1.17837792634964 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.17837792634964, + "total_car_to_hidden_at_8": 1.1859291791915894 }, "1665739511/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1665739511/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2508232858898205, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1747178030/train0": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15825371494561297, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8536934107542038, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1747178030/train1": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.027157030999660492, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.3089551740932353, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11134227365255356, + "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08418524265289307, + "selected_utility_gain_over_base_at_8": -0.06212134659290314, "selector_regret_at_8": 0.027157030999660492, - "support_gap_at_8": 1.8886577263474464 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8886577263474464, + "total_car_to_hidden_at_8": 1.915814757347107 }, "1747178030/train2": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, "branch_car_at_8": 0.034092940390110016, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.08595818214364334, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12480536848306656, + "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09071242809295654, + "selected_utility_gain_over_base_at_8": -0.05559416115283966, "selector_regret_at_8": 0.034092940390110016, - "support_gap_at_8": 1.8751946315169334 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8751946315169334, + "total_car_to_hidden_at_8": 1.9092875719070435 }, "1881223807/train0": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.020236089825630188, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, "selector_regret_at_8": 0.020236089825630188, - "support_gap_at_8": 1.9486478716135025 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9486478716135025, + "total_car_to_hidden_at_8": 1.9688839614391327 }, "1881223807/train1": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.009028241038322449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13674585276395504, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.034886181354522705, + "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.025857940316200256, + "selected_utility_gain_over_base_at_8": -0.006169050931930542, "selector_regret_at_8": 0.009028241038322449, - "support_gap_at_8": 1.9651138186454773 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9651138186454773, + "total_car_to_hidden_at_8": 1.9741420596837997 }, "1881223807/train2": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, "branch_car_at_8": 0.024854734539985657, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2146835718150399, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.043482705950737, + "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018627971410751343, + "selected_utility_gain_over_base_at_8": -0.013399019837379456, "selector_regret_at_8": 0.024854734539985657, - "support_gap_at_8": 1.956517294049263 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.956517294049263, + "total_car_to_hidden_at_8": 1.9813720285892487 }, "1919198188/train0": { + "base_success": 0.0, + "base_utility": 0.037198856472969055, "branch_car_at_8": 0.016854479908943176, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.15779226512372985, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08992795646190643, + "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07307347655296326, + "selected_utility_gain_over_base_at_8": 0.0358746200799942, "selector_regret_at_8": 0.016854479908943176, - "support_gap_at_8": 1.9100720435380936 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9100720435380936, + "total_car_to_hidden_at_8": 1.9269265234470367 }, "1919198188/train1": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13114443223224567, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.046244338154792786, + "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046244338154792786, + "selected_utility_gain_over_base_at_8": 0.009045466780662537, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.9537556618452072 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9537556618452072, + "total_car_to_hidden_at_8": 1.9537556618452072 }, "1919198188/train2": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, "branch_car_at_8": 0.018839523196220398, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.1669540555093941, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06275103986263275, + "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043911516666412354, + "selected_utility_gain_over_base_at_8": 0.0067126452922821045, "selector_regret_at_8": 0.018839523196220398, - "support_gap_at_8": 1.9372489601373672 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9372489601373672, + "total_car_to_hidden_at_8": 1.9560884833335876 }, "1991414747/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.05287821364799471, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1991414747/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9863876849412918, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1306768270010748, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013612315058708191, + "selected_utility_gain_over_base_at_8": -1.9863876849412918, "selector_regret_at_8": 1.9863876849412918, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9863876849412918 }, "1991414747/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11896663871153935, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1997090716/train0": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 1.9691698104143143, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, "selector_regret_at_8": 1.9691698104143143, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9691698104143143 }, "1997090716/train1": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2554442550173647, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "1997090716/train2": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.14617959762414967, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2015956957/train0": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, "branch_car_at_8": 0.01770364921540022, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.18664994005668376, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.45124312303960323, + "proposal_oracle_utility_gain_over_base_at_8": 0.38412333317101, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.433539473824203, + "selected_utility_gain_over_base_at_8": 0.3664196839556098, "selector_regret_at_8": 0.01770364921540022, - "support_gap_at_8": 0.07863824404776096 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07863824404776096, + "total_car_to_hidden_at_8": 0.09634189326316119 }, "2015956957/train1": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, "branch_car_at_8": 0.022195855714380743, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.16317219295797403, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05629209652543068, + "proposal_oracle_utility_gain_over_base_at_8": -0.010827693343162536, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.034096240811049935, + "selected_utility_gain_over_base_at_8": -0.033023549057543275, "selector_regret_at_8": 0.022195855714380743, - "support_gap_at_8": 0.4735892705619335 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.2, + "success_total_car_to_hidden_at_8": 0.2, + "support_gap_at_8": 0.4735892705619335, + "total_car_to_hidden_at_8": 0.49578512627631427 }, "2015956957/train2": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, "branch_car_at_8": 0.005573729798197746, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.1431252893441326, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.43981083631515505, + "proposal_oracle_utility_gain_over_base_at_8": 0.3726910464465618, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.4342371065169573, + "selected_utility_gain_over_base_at_8": 0.36711731664836406, "selector_regret_at_8": 0.005573729798197746, - "support_gap_at_8": 0.09007053077220917 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.09007053077220917, + "total_car_to_hidden_at_8": 0.09564426057040691 }, "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.5908347219228745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.16771630653845432, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8130142092704773, + "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22217948734760284, + "selected_utility_gain_over_base_at_8": -0.6166549175977707, "selector_regret_at_8": 0.5908347219228745, - "support_gap_at_8": 1.1869857907295227 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1869857907295227, + "total_car_to_hidden_at_8": 1.7778205126523972 }, "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.6792904287576675, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07548939908426067, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8358442187309265, + "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15655378997325897, + "selected_utility_gain_over_base_at_8": -0.6822806149721146, "selector_regret_at_8": 0.6792904287576675, - "support_gap_at_8": 1.1641557812690735 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1641557812690735, + "total_car_to_hidden_at_8": 1.843446210026741 }, "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.13508468824583414, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1611655950546265, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "2141802816/train0": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.07543906569480896, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, "selector_regret_at_8": 0.07543906569480896, - "support_gap_at_8": 1.8752704411745071 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8752704411745071, + "total_car_to_hidden_at_8": 1.950709506869316 }, "2141802816/train1": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 0.019766509532928467, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07786951284417709, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09589323401451111, + "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07612672448158264, + "selected_utility_gain_over_base_at_8": 0.005319476127624512, "selector_regret_at_8": 0.019766509532928467, - "support_gap_at_8": 1.904106765985489 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.904106765985489, + "total_car_to_hidden_at_8": 1.9238732755184174 }, "2141802816/train2": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, "branch_car_at_8": 1.861636370420456, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09903289408743085, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13836362957954407, + "selected_utility_gain_over_base_at_8": 0.06755638122558594, "selector_regret_at_8": 1.861636370420456, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.861636370420456 }, "256770874/train0": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.10820980370044708, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.030709639081801327, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.24107472598552704, + "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13286492228507996, + "selected_utility_gain_over_base_at_8": -0.7228831350803375, "selector_regret_at_8": 0.10820980370044708, - "support_gap_at_8": 1.758925274014473 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.758925274014473, + "total_car_to_hidden_at_8": 1.86713507771492 }, "256770874/train1": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.7044661641120911, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.03135306008748828, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8125, + "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10803383588790894, + "selected_utility_gain_over_base_at_8": -0.7477142214775085, "selector_regret_at_8": 0.7044661641120911, - "support_gap_at_8": 1.1875 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1875, + "total_car_to_hidden_at_8": 1.891966164112091 }, "256770874/train2": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, "branch_car_at_8": 0.046216949820518494, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.046630940401554466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.21061499416828156, + "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16439804434776306, + "selected_utility_gain_over_base_at_8": -0.6913500130176544, "selector_regret_at_8": 0.046216949820518494, - "support_gap_at_8": 1.7893850058317184 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7893850058317184, + "total_car_to_hidden_at_8": 1.835601955652237 }, "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, "branch_car_at_8": 0.0016456245793960989, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.11925245416072927, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.011432985425926745, + "proposal_oracle_utility_gain_over_base_at_8": -0.09603888296987861, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.009787360846530646, + "selected_utility_gain_over_base_at_8": -0.09768450754927471, "selector_regret_at_8": 0.0016456245793960989, - "support_gap_at_8": 1.1023782094707713 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.1023782094707713, + "total_car_to_hidden_at_8": 1.1040238340501674 }, "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, "branch_car_at_8": 0.011409573198761791, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.24605586147071373, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03147180867381394, + "proposal_oracle_utility_gain_over_base_at_8": -0.07600005972199142, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.020062235475052148, + "selected_utility_gain_over_base_at_8": -0.08740963292075321, "selector_regret_at_8": 0.011409573198761791, - "support_gap_at_8": 1.082339386222884 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.082339386222884, + "total_car_to_hidden_at_8": 1.0937489594216458 }, "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, "branch_car_at_8": 0.020201382460072637, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.15535141304867436, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032865648390725255, + "proposal_oracle_utility_gain_over_base_at_8": -0.0746062200050801, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012664265930652618, + "selected_utility_gain_over_base_at_8": -0.09480760246515274, "selector_regret_at_8": 0.020201382460072637, - "support_gap_at_8": 1.0809455465059727 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.0809455465059727, + "total_car_to_hidden_at_8": 1.1011469289660454 }, "416183106/train0": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 0.3360496461391449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10129212810935397, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5778142213821411, + "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.24176457524299622, + "selected_utility_gain_over_base_at_8": 0.041438162326812744, "selector_regret_at_8": 0.3360496461391449, - "support_gap_at_8": 1.4221857786178589 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4221857786178589, + "total_car_to_hidden_at_8": 1.7582354247570038 }, "416183106/train1": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.7668711841106415, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07978206783428046, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23312881588935852, + "selected_utility_gain_over_base_at_8": 0.03280240297317505, "selector_regret_at_8": 1.7668711841106415, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7668711841106415 }, "416183106/train2": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, "branch_car_at_8": 1.812951534986496, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18072825209943494, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18704846501350403, + "selected_utility_gain_over_base_at_8": -0.013277947902679443, "selector_regret_at_8": 1.812951534986496, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.812951534986496 }, "534717141/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9274959191679955, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.6159370955924921, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07250408083200455, + "selected_utility_gain_over_base_at_8": -1.9274959191679955, "selector_regret_at_8": 1.9274959191679955, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9274959191679955 }, "534717141/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 1.9558932781219482, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14369576234023818, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.04410672187805176, + "selected_utility_gain_over_base_at_8": -1.9558932781219482, "selector_regret_at_8": 1.9558932781219482, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9558932781219482 }, "534717141/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2870871533378466, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "551504635/train0": { + "base_success": 0.0, + "base_utility": 0.7239725589752197, "branch_car_at_8": 0.033253490924835205, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.10932818664818449, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7250897884368896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6918362975120544, + "selected_utility_gain_over_base_at_8": -0.03213626146316528, "selector_regret_at_8": 0.033253490924835205, - "support_gap_at_8": 1.2749102115631104 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2749102115631104, + "total_car_to_hidden_at_8": 1.3081637024879456 }, "551504635/train1": { + "base_success": 0.0, + "base_utility": 0.7247228622436523, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2299325778712667, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7224790453910828, + "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7224790453910828, + "selected_utility_gain_over_base_at_8": -0.00224381685256958, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.2775209546089172 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2775209546089172, + "total_car_to_hidden_at_8": 1.2775209546089172 }, "551504635/train2": { + "base_success": 0.0, + "base_utility": 0.7238867878913879, "branch_car_at_8": 1.273692786693573, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2702494805053396, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.726307213306427, + "selected_utility_gain_over_base_at_8": 0.0024204254150390625, "selector_regret_at_8": 1.273692786693573, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.273692786693573 }, "565369392/train0": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, "branch_car_at_8": 0.06443698455890019, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.22308845281795123, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7852714310089747, + "proposal_oracle_utility_gain_over_base_at_8": -0.014494215448697409, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7208344464500746, + "selected_utility_gain_over_base_at_8": -0.07893120000759761, "selector_regret_at_8": 0.06443698455890019, - "support_gap_at_8": 0.6413601487874985 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.6413601487874985, + "total_car_to_hidden_at_8": 0.7057971333463987 }, "565369392/train1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, "branch_car_at_8": 0.046260833740234375, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.13991651305345584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.17366480827331543, + "proposal_oracle_utility_gain_over_base_at_8": -0.6261008381843567, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.12740397453308105, + "selected_utility_gain_over_base_at_8": -0.6723616719245911, "selector_regret_at_8": 0.046260833740234375, - "support_gap_at_8": 1.2529667715231578 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2529667715231578, + "total_car_to_hidden_at_8": 1.2992276052633922 }, "565369392/train2": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, "branch_car_at_8": 0.04557279869914055, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.17390197648626904, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.1326704372962316, + "proposal_oracle_utility_gain_over_base_at_8": -0.6670952091614405, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.08709763859709103, + "selected_utility_gain_over_base_at_8": -0.712668007860581, "selector_regret_at_8": 0.04557279869914055, - "support_gap_at_8": 1.2939611425002415 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2939611425002415, + "total_car_to_hidden_at_8": 1.339533941199382 }, "565621826/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "565621826/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "565621826/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, "branch_car_at_8": 0.043168901621053614, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, "outcome_ptr_at_8": 0.3333333333333333, "pairwise_causal_calibration_ece": 0.10391809443539522, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05598785417775313, + "proposal_oracle_utility_gain_over_base_at_8": -0.019304243847727776, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012818952556699514, + "selected_utility_gain_over_base_at_8": -0.06247314546878139, "selector_regret_at_8": 0.043168901621053614, - "support_gap_at_8": 0.1816591378301382 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1816591378301382, + "total_car_to_hidden_at_8": 0.2248280394511918 }, "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, "branch_car_at_8": 0.012068324722349644, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07276320034061332, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032918572425842285, + "proposal_oracle_utility_gain_over_base_at_8": -0.04237352559963862, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02085024770349264, + "selected_utility_gain_over_base_at_8": -0.05444185032198826, "selector_regret_at_8": 0.012068324722349644, - "support_gap_at_8": 0.20472841958204904 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.20472841958204904, + "total_car_to_hidden_at_8": 0.2167967443043987 }, "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, "branch_car_at_8": 0.011032561771571636, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.2625659187813512, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07848061621189117, + "proposal_oracle_utility_gain_over_base_at_8": 0.0031885181864102683, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06744805444031954, + "selected_utility_gain_over_base_at_8": -0.007844043585161367, "selector_regret_at_8": 0.011032561771571636, - "support_gap_at_8": 0.15916637579600015 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.15916637579600015, + "total_car_to_hidden_at_8": 0.1701989375675718 }, "626486813/train0": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, "branch_car_at_8": 0.018570441869087517, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.16162412565990822, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5615248780231923, + "proposal_oracle_utility_gain_over_base_at_8": 0.0560848613968119, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5429544361541048, + "selected_utility_gain_over_base_at_8": 0.037514419527724385, "selector_regret_at_8": 0.018570441869087517, - "support_gap_at_8": 0.0801479306537658 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0801479306537658, + "total_car_to_hidden_at_8": 0.08356922247912735 }, "626486813/train1": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, "branch_car_at_8": 0.02500777377281338, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.2420589895144555, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07365635968744755, + "proposal_oracle_utility_gain_over_base_at_8": -0.43178365693893284, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.04864858591463417, + "selected_utility_gain_over_base_at_8": -0.4567914307117462, "selector_regret_at_8": 0.02500777377281338, - "support_gap_at_8": 0.5414478182792664 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5414478182792664, + "total_car_to_hidden_at_8": 0.5638198243686929 }, "626486813/train2": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, "branch_car_at_8": 0.031722720130346715, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.23561688988950158, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07003506645560265, + "proposal_oracle_utility_gain_over_base_at_8": -0.43540495017077774, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.03831234632525593, + "selected_utility_gain_over_base_at_8": -0.46712767030112445, "selector_regret_at_8": 0.031722720130346715, - "support_gap_at_8": 0.5432130172848701 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5432130172848701, + "total_car_to_hidden_at_8": 0.5741560639580712 }, "636680137/train0": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.1699944666253151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012414653785526752, + "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.012414653785526752, + "selected_utility_gain_over_base_at_8": -1.9875853462144732, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.9875853462144732 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9875853462144732, + "total_car_to_hidden_at_8": 1.9875853462144732 }, "636680137/train1": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.07628525991805817, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.013928472995758057, + "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013928472995758057, + "selected_utility_gain_over_base_at_8": -1.986071527004242, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.986071527004242 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.986071527004242, + "total_car_to_hidden_at_8": 1.986071527004242 }, "636680137/train2": { + "base_success": 1.0, + "base_utility": 2.0, "branch_car_at_8": 0.006491700187325478, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.12019361687075947, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.02436021901667118, + "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.017868518829345703, + "selected_utility_gain_over_base_at_8": -1.9821314811706543, "selector_regret_at_8": 0.006491700187325478, - "support_gap_at_8": 1.9756397809833288 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9756397809833288, + "total_car_to_hidden_at_8": 1.9821314811706543 }, "706479600/train0": { + "base_success": 0.0, + "base_utility": 0.8125406503677368, "branch_car_at_8": 1.1865743398666382, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0922387871049479, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8134256601333618, + "selected_utility_gain_over_base_at_8": 0.000885009765625, "selector_regret_at_8": 1.1865743398666382, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.1865743398666382 }, "706479600/train1": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.16443847051324711, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "706479600/train2": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.22468613298280815, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, "branch_car_at_8": 0.07732226233929396, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.14360585133877676, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1069931760430336, + "proposal_oracle_utility_gain_over_base_at_8": -0.004540383815765381, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.029670913703739643, + "selected_utility_gain_over_base_at_8": -0.08186264615505934, "selector_regret_at_8": 0.07732226233929396, - "support_gap_at_8": 0.12881441414356232 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12881441414356232, + "total_car_to_hidden_at_8": 0.20613667648285627 }, "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, "branch_car_at_8": 0.02701035887002945, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.2344413298276417, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07342664152383804, + "proposal_oracle_utility_gain_over_base_at_8": -0.03810691833496094, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046416282653808594, + "selected_utility_gain_over_base_at_8": -0.06511727720499039, "selector_regret_at_8": 0.02701035887002945, - "support_gap_at_8": 0.16238094866275787 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.16238094866275787, + "total_car_to_hidden_at_8": 0.18939130753278732 }, "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, "branch_car_at_8": 0.01770863775163889, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, "outcome_ptr_at_8": 0.0, "pairwise_causal_calibration_ece": 0.14149358436045273, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0669428389519453, + "proposal_oracle_utility_gain_over_base_at_8": -0.044590720906853676, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.049234201200306416, + "selected_utility_gain_over_base_at_8": -0.062299358658492565, "selector_regret_at_8": 0.01770863775163889, - "support_gap_at_8": 0.1688647512346506 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1688647512346506, + "total_car_to_hidden_at_8": 0.1865733889862895 }, "75070641/train0": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.07482179999351501, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.2497251397544613, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2396746575832367, + "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16485285758972168, + "selected_utility_gain_over_base_at_8": -0.03789877891540527, "selector_regret_at_8": 0.07482179999351501, - "support_gap_at_8": 1.7603253424167633 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7603253424167633, + "total_car_to_hidden_at_8": 1.8351471424102783 }, "75070641/train1": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.5863077491521835, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.18559462278331035, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8128874897956848, + "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22657974064350128, + "selected_utility_gain_over_base_at_8": 0.02382810413837433, "selector_regret_at_8": 0.5863077491521835, - "support_gap_at_8": 1.1871125102043152 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1871125102043152, + "total_car_to_hidden_at_8": 1.7734202593564987 }, "75070641/train2": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, "branch_car_at_8": 0.3710349500179291, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.09051828052202912, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5650836229324341, + "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.194048672914505, + "selected_utility_gain_over_base_at_8": -0.008702963590621948, "selector_regret_at_8": 0.3710349500179291, - "support_gap_at_8": 1.434916377067566 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.434916377067566, + "total_car_to_hidden_at_8": 1.805951327085495 }, "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, "branch_car_at_8": 0.015960504300892353, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, "outcome_ptr_at_8": 0.25, "pairwise_causal_calibration_ece": 0.21088094062197144, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5703969299793243, + "proposal_oracle_utility_gain_over_base_at_8": 0.022872742265462875, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.554436425678432, + "selected_utility_gain_over_base_at_8": 0.006912237964570522, "selector_regret_at_8": 0.015960504300892353, - "support_gap_at_8": 0.05905248410999775 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05905248410999775, + "total_car_to_hidden_at_8": 0.0750129884108901 }, "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, "branch_car_at_8": 0.04561196081340313, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.07531396139335009, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11342880013398826, + "proposal_oracle_utility_gain_over_base_at_8": -0.4340953875798732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.06781683932058513, + "selected_utility_gain_over_base_at_8": -0.47970734839327633, "selector_regret_at_8": 0.04561196081340313, - "support_gap_at_8": 0.08526064571924508 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.08526064571924508, + "total_car_to_hidden_at_8": 0.12081881728954613 }, "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, "branch_car_at_8": 0.024658456211909652, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, "outcome_ptr_at_8": 0.75, "pairwise_causal_calibration_ece": 0.14269822176062383, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11239026742987335, + "proposal_oracle_utility_gain_over_base_at_8": -0.4351339202839881, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.0877318112179637, + "selected_utility_gain_over_base_at_8": -0.45979237649589777, "selector_regret_at_8": 0.024658456211909652, - "support_gap_at_8": 0.07677002181299031 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07677002181299031, + "total_car_to_hidden_at_8": 0.10090384539216757 }, "77147264/train0": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, "branch_car_at_8": 1.868594542145729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.06512707818376468, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.5, + "selected_utility_at_8": 0.13140545785427094, + "selected_utility_gain_over_base_at_8": -1.2901740223169327, "selector_regret_at_8": 1.868594542145729, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.868594542145729 }, "77147264/train1": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.2630803856693186, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.426636129617691, + "proposal_oracle_utility_gain_over_base_at_8": 0.005056649446487427, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.426636129617691, + "selected_utility_gain_over_base_at_8": 0.005056649446487427, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.573363870382309 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.573363870382309, + "total_car_to_hidden_at_8": 0.573363870382309 }, "77147264/train2": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.12309184879211332, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5784205198287964, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "794181626/train0": { + "base_success": 0.0, + "base_utility": 0.1733371466398239, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.12119924562800825, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.826662853360176, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "794181626/train1": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28339362113630795, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "794181626/train2": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, "selector_regret_at_8": 0.0, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 }, "957101489/train0": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.20250855048512742, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5881698727607727, + "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5881698727607727, + "selected_utility_gain_over_base_at_8": 0.3842049241065979, "selector_regret_at_8": 0.0, - "support_gap_at_8": 1.4118301272392273 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4118301272392273, + "total_car_to_hidden_at_8": 1.4118301272392273 }, "957101489/train1": { + "base_success": 0.0, + "base_utility": 0.20372186601161957, "branch_car_at_8": 1.7757539749145508, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.07643021229779011, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22424602508544922, + "selected_utility_gain_over_base_at_8": 0.02052415907382965, "selector_regret_at_8": 1.7757539749145508, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7757539749145508 }, "957101489/train2": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, "branch_car_at_8": 1.432861328125, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.21219243624964876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.567138671875, + "selected_utility_gain_over_base_at_8": 0.3631737232208252, "selector_regret_at_8": 1.432861328125, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.432861328125 }, "998761162/train0": { + "base_success": 0.0, + "base_utility": 0.0689050555229187, "branch_car_at_8": 0.07749588787555695, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.23606164078349207, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08403940498828888, + "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006543517112731934, + "selected_utility_gain_over_base_at_8": -0.06236153841018677, "selector_regret_at_8": 0.07749588787555695, - "support_gap_at_8": 1.9159605950117111 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9159605950117111, + "total_car_to_hidden_at_8": 1.993456482887268 }, "998761162/train1": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.9512543678283691, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.28991947553571784, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04874563217163086, + "selected_utility_gain_over_base_at_8": -0.019070938229560852, "selector_regret_at_8": 1.9512543678283691, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9512543678283691 }, "998761162/train2": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, "branch_car_at_8": 1.939187914133072, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 1.0, "pairwise_causal_calibration_ece": 0.11706618974004251, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0608120858669281, + "selected_utility_gain_over_base_at_8": -0.007004484534263611, "selector_regret_at_8": 1.939187914133072, - "support_gap_at_8": 0.0 + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.939187914133072 } } diff --git a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_task.json b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_task.json index 95aa64b1831303f4f09a6dd9febdc98059773735..54ab0b7dda30c9a1c5314aec27032079a5f8586c 100644 --- a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_task.json +++ b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/metrics_by_task.json @@ -1,37 +1,117 @@ { "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.0669810656044219, "branch_car_at_8": 0.516516399052408, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.5, "pairwise_causal_calibration_ece": 0.1608032528409541, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.16666666666666666, + "proposal_oracle_utility_at_8": 1.3387502448426352, + "proposal_oracle_utility_gain_over_base_at_8": 0.2717691792382134, + "selected_success_at_8": 0.2222222222222222, + "selected_success_gain_over_base_at_8": -0.1111111111111111, + "selected_utility_at_8": 0.8222338457902273, + "selected_utility_gain_over_base_at_8": -0.24474721981419456, "selector_regret_at_8": 0.516516399052408, - "support_gap_at_8": 0.6612497551573647 + "success_selector_gap_at_8": 0.2777777777777778, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.7777777777777778, + "support_gap_at_8": 0.6612497551573647, + "total_car_to_hidden_at_8": 1.1777661542097728 }, "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, "branch_car_at_8": 0.2721092589199543, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6666666666666666, "pairwise_causal_calibration_ece": 0.17559224662436113, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.3654347012440364, + "proposal_oracle_utility_gain_over_base_at_8": 0.2120050440231959, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09332544232408206, + "selected_utility_gain_over_base_at_8": -0.0601042148967584, "selector_regret_at_8": 0.2721092589199543, - "support_gap_at_8": 1.6345652987559636 + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.6345652987559636, + "total_car_to_hidden_at_8": 1.9066745576759179 }, "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, "branch_car_at_8": 0.05991193203954026, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, "outcome_ptr_at_8": 0.3235294117647059, "pairwise_causal_calibration_ece": 0.1645321267392569, + "proposal_oracle_success_at_8": 0.18627450980392157, + "proposal_oracle_success_gain_over_base_at_8": -0.10784313725490197, + "proposal_oracle_utility_at_8": 0.4324846717342963, + "proposal_oracle_utility_gain_over_base_at_8": -0.21874524904905782, + "selected_success_at_8": 0.16666666666666666, + "selected_success_gain_over_base_at_8": -0.12745098039215685, + "selected_utility_at_8": 0.372572739694756, + "selected_utility_gain_over_base_at_8": -0.2786571810885981, "selector_regret_at_8": 0.05991193203954026, - "support_gap_at_8": 0.3689149604584136 + "success_selector_gap_at_8": 0.0196078431372549, + "success_support_gap_at_8": 0.14705882352941177, + "success_total_car_to_hidden_at_8": 0.16666666666666666, + "support_gap_at_8": 0.3689149604584136, + "total_car_to_hidden_at_8": 0.42758709733321915 }, "PushCube-v1": { + "base_success": 0.358974358974359, + "base_utility": 0.7709262684369699, "branch_car_at_8": 0.4573507461792383, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6153846153846154, "pairwise_causal_calibration_ece": 0.16409670072723917, + "proposal_oracle_success_at_8": 0.7692307692307693, + "proposal_oracle_success_gain_over_base_at_8": 0.41025641025641024, + "proposal_oracle_utility_at_8": 1.5530982193274376, + "proposal_oracle_utility_gain_over_base_at_8": 0.7821719508904678, + "selected_success_at_8": 0.5384615384615384, + "selected_success_gain_over_base_at_8": 0.1794871794871795, + "selected_utility_at_8": 1.0957474731481993, + "selected_utility_gain_over_base_at_8": 0.3248212047112294, "selector_regret_at_8": 0.4573507461792383, - "support_gap_at_8": 0.4469017806725624 + "success_selector_gap_at_8": 0.23076923076923078, + "success_support_gap_at_8": 0.23076923076923078, + "success_total_car_to_hidden_at_8": 0.46153846153846156, + "support_gap_at_8": 0.4469017806725624, + "total_car_to_hidden_at_8": 0.9042525268518008 }, "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8034174028370116, "branch_car_at_8": 0.593565668993526, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, "outcome_ptr_at_8": 0.6, "pairwise_causal_calibration_ece": 0.1529426346924437, + "proposal_oracle_success_at_8": 0.4444444444444444, + "proposal_oracle_success_gain_over_base_at_8": 0.24444444444444444, + "proposal_oracle_utility_at_8": 1.1668234914541245, + "proposal_oracle_utility_gain_over_base_at_8": 0.3634060886171129, + "selected_success_at_8": 0.17777777777777778, + "selected_success_gain_over_base_at_8": -0.022222222222222223, + "selected_utility_at_8": 0.5732578224605984, + "selected_utility_gain_over_base_at_8": -0.23015958037641313, "selector_regret_at_8": 0.593565668993526, - "support_gap_at_8": 0.8331765085458755 + "success_selector_gap_at_8": 0.26666666666666666, + "success_support_gap_at_8": 0.5555555555555556, + "success_total_car_to_hidden_at_8": 0.8222222222222222, + "support_gap_at_8": 0.8331765085458755, + "total_car_to_hidden_at_8": 1.4267421775394016 } } diff --git a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/report.md b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/report.md index 1571e5ea7268cceebfa2a897a0afbf448bd4781f..14e1e89808f91be2e0b37888aa926342a344b738 100644 --- a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/report.md +++ b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/report.md @@ -4,8 +4,24 @@ K: `8` | Metric | N | Micro mean | 95% CI | Task macro | Seed macro | | --- | ---: | ---: | ---: | ---: | ---: | -| branch_car_at_8 | 207 | 0.2936 | [0.2131, 0.3721] | 0.3799 | 0.4056 | -| outcome_ptr_at_8 | 207 | 0.4589 | [0.3913, 0.5266] | 0.5411 | 0.4850 | -| pairwise_causal_calibration_ece | 207 | 0.1618 | [0.1463, 0.1771] | 0.1636 | 0.1579 | -| selector_regret_at_8 | 207 | 0.2936 | [0.2131, 0.3721] | 0.3799 | 0.4056 | -| support_gap_at_8 | 207 | 0.5283 | [0.4234, 0.6341] | 0.7890 | 0.6062 | +| base_success | 207 | 0.2850 | [0.2271, 0.3478] | 0.2373 | 0.3227 | +| base_utility | 207 | 0.7358 | [0.6279, 0.8512] | 0.6892 | 0.8424 | +| branch_car_at_8 | 207 | 0.2936 | [0.2100, 0.3738] | 0.3799 | 0.4056 | +| hidden_chart_oracle_success_at_8 | 207 | 0.6667 | [0.6087, 0.7246] | 0.8647 | 0.8358 | +| hidden_chart_oracle_utility_at_8 | 207 | 1.4000 | [1.2940, 1.5057] | 1.7565 | 1.7062 | +| outcome_ptr_at_8 | 207 | 0.4589 | [0.3913, 0.5314] | 0.5411 | 0.4850 | +| pairwise_causal_calibration_ece | 207 | 0.1618 | [0.1468, 0.1771] | 0.1636 | 0.1579 | +| proposal_oracle_success_at_8 | 207 | 0.3768 | [0.3092, 0.4444] | 0.3800 | 0.4757 | +| proposal_oracle_success_gain_over_base_at_8 | 207 | 0.0918 | [0.0193, 0.1594] | 0.1427 | 0.1530 | +| proposal_oracle_utility_at_8 | 207 | 0.8811 | [0.7586, 1.0042] | 0.9713 | 1.1035 | +| proposal_oracle_utility_gain_over_base_at_8 | 207 | 0.1453 | [0.0316, 0.2667] | 0.2821 | 0.2611 | +| selected_success_at_8 | 207 | 0.2415 | [0.1884, 0.3043] | 0.2210 | 0.2842 | +| selected_success_gain_over_base_at_8 | 207 | -0.0435 | [-0.1111, 0.0242] | -0.0163 | -0.0385 | +| selected_utility_at_8 | 207 | 0.5875 | [0.4854, 0.7024] | 0.5914 | 0.6978 | +| selected_utility_gain_over_base_at_8 | 207 | -0.1483 | [-0.2654, -0.0237] | -0.0978 | -0.1446 | +| selector_regret_at_8 | 207 | 0.2936 | [0.2100, 0.3738] | 0.3799 | 0.4056 | +| success_selector_gap_at_8 | 207 | 0.1353 | [0.0870, 0.1884] | 0.1590 | 0.1915 | +| success_support_gap_at_8 | 207 | 0.2947 | [0.2319, 0.3623] | 0.4867 | 0.3619 | +| success_total_car_to_hidden_at_8 | 207 | 0.4300 | [0.3575, 0.4976] | 0.6456 | 0.5534 | +| support_gap_at_8 | 207 | 0.5283 | [0.4234, 0.6411] | 0.7890 | 0.6062 | +| total_car_to_hidden_at_8 | 207 | 0.8213 | [0.6955, 0.9329] | 1.1686 | 1.0116 | diff --git a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/table.tex b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/table.tex index 31750a01692c07356f768a96c9ae4da98e65563e..40a7939a99f88e3c4661ccb20fbde320c103f133 100644 --- a/workspace/runs/ctt_val_rollout_comparison/measured_metrics/table.tex +++ b/workspace/runs/ctt_val_rollout_comparison/measured_metrics/table.tex @@ -3,10 +3,26 @@ \toprule Metric & N & Micro mean & CI low & CI high \\ \midrule -branch\_car\_at\_8 & 207 & 0.2936 & 0.2131 & 0.3721 \\ -outcome\_ptr\_at\_8 & 207 & 0.4589 & 0.3913 & 0.5266 \\ -pairwise\_causal\_calibration\_ece & 207 & 0.1618 & 0.1463 & 0.1771 \\ -selector\_regret\_at\_8 & 207 & 0.2936 & 0.2131 & 0.3721 \\ -support\_gap\_at\_8 & 207 & 0.5283 & 0.4234 & 0.6341 \\ +base\_success & 207 & 0.2850 & 0.2271 & 0.3478 \\ +base\_utility & 207 & 0.7358 & 0.6279 & 0.8512 \\ +branch\_car\_at\_8 & 207 & 0.2936 & 0.2100 & 0.3738 \\ +hidden\_chart\_oracle\_success\_at\_8 & 207 & 0.6667 & 0.6087 & 0.7246 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 207 & 1.4000 & 1.2940 & 1.5057 \\ +outcome\_ptr\_at\_8 & 207 & 0.4589 & 0.3913 & 0.5314 \\ +pairwise\_causal\_calibration\_ece & 207 & 0.1618 & 0.1468 & 0.1771 \\ +proposal\_oracle\_success\_at\_8 & 207 & 0.3768 & 0.3092 & 0.4444 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 207 & 0.0918 & 0.0193 & 0.1594 \\ +proposal\_oracle\_utility\_at\_8 & 207 & 0.8811 & 0.7586 & 1.0042 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 207 & 0.1453 & 0.0316 & 0.2667 \\ +selected\_success\_at\_8 & 207 & 0.2415 & 0.1884 & 0.3043 \\ +selected\_success\_gain\_over\_base\_at\_8 & 207 & -0.0435 & -0.1111 & 0.0242 \\ +selected\_utility\_at\_8 & 207 & 0.5875 & 0.4854 & 0.7024 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 207 & -0.1483 & -0.2654 & -0.0237 \\ +selector\_regret\_at\_8 & 207 & 0.2936 & 0.2100 & 0.3738 \\ +success\_selector\_gap\_at\_8 & 207 & 0.1353 & 0.0870 & 0.1884 \\ +success\_support\_gap\_at\_8 & 207 & 0.2947 & 0.2319 & 0.3623 \\ +success\_total\_car\_to\_hidden\_at\_8 & 207 & 0.4300 & 0.3575 & 0.4976 \\ +support\_gap\_at\_8 & 207 & 0.5283 & 0.4234 & 0.6411 \\ +total\_car\_to\_hidden\_at\_8 & 207 & 0.8213 & 0.6955 & 0.9329 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_val_rollout_comparison/metrics.json b/workspace/runs/ctt_val_rollout_comparison/metrics.json index dca94f95acd8ed9f52e57162db8c23eb232508e6..930d9a75f421c4858f0050028129fa943b671769 100644 --- a/workspace/runs/ctt_val_rollout_comparison/metrics.json +++ b/workspace/runs/ctt_val_rollout_comparison/metrics.json @@ -8,16 +8,399 @@ "runs/ctt_residual_rollout_val69_seed2" ], "source_content_hash": "90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957", + "success_summary": { + "base_success_rate": 0.28502415458937197, + "base_utility_mean": 0.7358034186915535, + "hidden_chart_oracle_success_rate": 0.6666666666666666, + "hidden_chart_oracle_utility_mean": 1.3999846732054932, + "max_restore_error": 1.4901161193847656e-08, + "proposal_oracle_success_gain_over_base": 0.09178743961352658, + "proposal_oracle_success_rate": 0.37681159420289856, + "proposal_oracle_utility_mean": 0.8810875975700648, + "selected_success_gain_over_base": -0.043478260869565216, + "selected_success_rate": 0.24154589371980675, + "selected_utility_mean": 0.5875041471669958, + "success_selector_gap": 0.13526570048309178, + "success_support_gap": 0.2946859903381642 + }, "summary": { + "base_success": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.39775413711583923, + "low": 0.25236406619385343, + "mean": 0.322695035460993, + "n": 141, + "num_samples": 500 + }, + "mean": 0.322695035460993, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 0.0, + "1176943004/train1": 0.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 0.0, + "1231780997/train1": 0.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.5, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 0.0, + "1375492786/train0": 1.0, + "1375492786/train1": 1.0, + "1375492786/train2": 1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 0.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.6666666666666666, + "1549324805/train2": 0.6666666666666666, + "1607773860/train0": 1.0, + "1607773860/train1": 1.0, + "1607773860/train2": 1.0, + "1665739511/train0": 1.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 1.0, + "1991414747/train1": 1.0, + "1991414747/train2": 1.0, + "1997090716/train0": 0.0, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.0, + "2015956957/train1": 0.0, + "2015956957/train2": 0.0, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 0.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 1.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 0.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.3333333333333333, + "565369392/train2": 0.3333333333333333, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 0.0, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.25, + "767747424/train2": 0.25, + "77147264/train0": 0.5, + "77147264/train1": 0.5, + "77147264/train2": 0.5, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 0.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 0.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3306184012066365, + "low": 0.10666666666666666, + "mean": 0.23728506787330317, + "n": 5, + "num_samples": 500 + }, + "mean": 0.23728506787330317, + "per_task": { + "LiftPegUpright-v1": 0.3333333333333333, + "PickCube-v1": 0.0, + "PullCube-v1": 0.29411764705882354, + "PushCube-v1": 0.358974358974359, + "StackCube-v1": 0.2 + } + }, + "metric": "base_success", + "micro": { + "confidence": 0.95, + "high": 0.34782608695652173, + "low": 0.22705314009661837, + "mean": 0.28502415458937197, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "base_utility": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.9718880789716623, + "low": 0.7188909860138495, + "mean": 0.84238802050422, + "n": 141, + "num_samples": 500 + }, + "mean": 0.84238802050422, + "per_seed": { + "1025740212/train0": 0.528373658657074, + "1025740212/train1": 0.528373658657074, + "1025740212/train2": 0.528373658657074, + "1055157354/train0": 0.17649109661579132, + "1055157354/train1": 0.17649109661579132, + "1055157354/train2": 0.19648407399654388, + "1055567618/train0": 2.0, + "1055567618/train1": 2.0, + "1055567618/train2": 2.0, + "1113268940/train0": 0.42231252789497375, + "1113268940/train1": 0.42231252789497375, + "1113268940/train2": 0.42231252789497375, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 0.05094052851200104, + "1176943004/train1": 0.058033183217048645, + "1176943004/train2": 0.05094052851200104, + "1189392917/train0": 0.10841864347457886, + "1189392917/train1": 0.10841864347457886, + "1189392917/train2": 0.10841864347457886, + "1231780997/train0": 0.7270061373710632, + "1231780997/train1": 0.7270061373710632, + "1231780997/train2": 0.7270061373710632, + "1271718007/train0": 1.4142604768276215, + "1271718007/train1": 1.415381133556366, + "1271718007/train2": 1.415381133556366, + "1273247072/train0": 0.15342965722084045, + "1273247072/train1": 0.15342965722084045, + "1273247072/train2": 0.15342965722084045, + "1358421325/train0": 0.8327022790908813, + "1358421325/train1": 0.8324764966964722, + "1358421325/train2": 0.8327022790908813, + "1375492786/train0": 2.0, + "1375492786/train1": 2.0, + "1375492786/train2": 2.0, + "141984615/train0": 0.09381213908394177, + "141984615/train1": 0.09381213908394177, + "141984615/train2": 0.09381213908394177, + "1475440973/train0": 2.0, + "1475440973/train1": 2.0, + "1475440973/train2": 2.0, + "1483645336/train0": 0.01641605794429779, + "1483645336/train1": 2.0, + "1483645336/train2": 2.0, + "1522272686/train0": 0.19476109743118286, + "1522272686/train1": 0.19844311475753784, + "1522272686/train2": 0.19476109743118286, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 2.0, + "1549324805/train0": 1.3787474483251572, + "1549324805/train1": 1.3787474483251572, + "1549324805/train2": 1.3787474483251572, + "1607773860/train0": 2.0, + "1607773860/train1": 2.0, + "1607773860/train2": 2.0, + "1665739511/train0": 2.0, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 0.1463065892457962, + "1747178030/train1": 0.1463065892457962, + "1747178030/train2": 0.1463065892457962, + "1881223807/train0": 0.0320269912481308, + "1881223807/train1": 0.0320269912481308, + "1881223807/train2": 0.0320269912481308, + "1919198188/train0": 0.037198856472969055, + "1919198188/train1": 0.03719887137413025, + "1919198188/train2": 0.03719887137413025, + "1991414747/train0": 2.0, + "1991414747/train1": 2.0, + "1991414747/train2": 2.0, + "1997090716/train0": 0.13927479088306427, + "1997090716/train1": 0.13927479088306427, + "1997090716/train2": 0.13927479088306427, + "2015956957/train0": 0.06711978986859321, + "2015956957/train1": 0.06711978986859321, + "2015956957/train2": 0.06711978986859321, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 0.8388344049453735, + "211245842/train1": 0.8388344049453735, + "211245842/train2": 0.8388344049453735, + "2141802816/train0": 0.07080724835395813, + "2141802816/train1": 0.07080724835395813, + "2141802816/train2": 0.07080724835395813, + "256770874/train0": 0.8557480573654175, + "256770874/train1": 0.8557480573654175, + "256770874/train2": 0.8557480573654175, + "278008524/train0": 0.10747186839580536, + "278008524/train1": 0.10747186839580536, + "278008524/train2": 0.10747186839580536, + "416183106/train0": 0.20032641291618347, + "416183106/train1": 0.20032641291618347, + "416183106/train2": 0.20032641291618347, + "534717141/train0": 2.0, + "534717141/train1": 2.0, + "534717141/train2": 2.0, + "551504635/train0": 0.7239725589752197, + "551504635/train1": 0.7247228622436523, + "551504635/train2": 0.7238867878913879, + "565369392/train0": 0.7997656464576721, + "565369392/train1": 0.7997656464576721, + "565369392/train2": 0.7997656464576721, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.07529209802548091, + "585904130/train1": 0.07529209802548091, + "585904130/train2": 0.07529209802548091, + "626486813/train0": 0.5054400166263804, + "626486813/train1": 0.5054400166263804, + "626486813/train2": 0.5054400166263804, + "636680137/train0": 2.0, + "636680137/train1": 2.0, + "636680137/train2": 2.0, + "706479600/train0": 0.8125406503677368, + "706479600/train1": 0.8350887894630432, + "706479600/train2": 0.8350887894630432, + "750448296/train0": 0.11153355985879898, + "750448296/train1": 0.11153355985879898, + "750448296/train2": 0.11153355985879898, + "75070641/train0": 0.20275163650512695, + "75070641/train1": 0.20275163650512695, + "75070641/train2": 0.20275163650512695, + "767747424/train0": 0.5475241877138615, + "767747424/train1": 0.5475241877138615, + "767747424/train2": 0.5475241877138615, + "77147264/train0": 1.4215794801712036, + "77147264/train1": 1.4215794801712036, + "77147264/train2": 1.4215794801712036, + "794181626/train0": 0.1733371466398239, + "794181626/train1": 0.18095605075359344, + "794181626/train2": 0.18095605075359344, + "957101489/train0": 0.2039649486541748, + "957101489/train1": 0.20372186601161957, + "957101489/train2": 0.2039649486541748, + "998761162/train0": 0.0689050555229187, + "998761162/train1": 0.06781657040119171, + "998761162/train2": 0.06781657040119171 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.9246198772067178, + "low": 0.4069265285873006, + "mean": 0.6891968629765195, + "n": 5, + "num_samples": 500 + }, + "mean": 0.6891968629765195, + "per_task": { + "LiftPegUpright-v1": 1.0669810656044219, + "PickCube-v1": 0.15342965722084045, + "PullCube-v1": 0.6512299207833541, + "PushCube-v1": 0.7709262684369699, + "StackCube-v1": 0.8034174028370116 + } + }, + "metric": "base_utility", + "micro": { + "confidence": 0.95, + "high": 0.851249745030154, + "low": 0.6279041040598771, + "mean": 0.7358034186915535, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, "branch_car_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5194647902708409, - "low": 0.2999603762294949, + "high": 0.5224681463797662, + "low": 0.2913920145155972, "mean": 0.40561739670197494, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.40561739670197494, "per_seed": { @@ -167,11 +550,11 @@ "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.5319262530406317, - "low": 0.19367229081819662, + "high": 0.5390796998678109, + "low": 0.18723032816778867, "mean": 0.3798908010369334, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.3798908010369334, "per_task": { @@ -185,11 +568,379 @@ "metric": "branch_car_at_8", "micro": { "confidence": 0.95, - "high": 0.3721372773162211, - "low": 0.21313571063115527, + "high": 0.37381450664971905, + "low": 0.2099735496069669, "mean": 0.293583450403069, "n": 207, - "num_samples": 1000 + "num_samples": 500 + }, + "n": 207 + }, + "hidden_chart_oracle_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8884160756501184, + "low": 0.7735224586288418, + "mean": 0.8358156028368796, + "n": 141, + "num_samples": 500 + }, + "mean": 0.8358156028368796, + "per_seed": { + "1025740212/train0": 1.0, + "1025740212/train1": 1.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 1.0, + "1055157354/train2": 1.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 1.0, + "1113268940/train1": 1.0, + "1113268940/train2": 1.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 1.0, + "1176943004/train1": 1.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 1.0, + "1231780997/train1": 1.0, + "1231780997/train2": 1.0, + "1271718007/train0": 1.0, + "1271718007/train1": 1.0, + "1271718007/train2": 1.0, + "1273247072/train0": 1.0, + "1273247072/train1": 1.0, + "1273247072/train2": 1.0, + "1358421325/train0": 1.0, + "1358421325/train1": 1.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 1.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.6666666666666666, + "1549324805/train2": 0.6666666666666666, + "1607773860/train0": 1.0, + "1607773860/train1": 1.0, + "1607773860/train2": 1.0, + "1665739511/train0": 1.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1881223807/train0": 1.0, + "1881223807/train1": 1.0, + "1881223807/train2": 1.0, + "1919198188/train0": 1.0, + "1919198188/train1": 1.0, + "1919198188/train2": 1.0, + "1991414747/train0": 1.0, + "1991414747/train1": 1.0, + "1991414747/train2": 1.0, + "1997090716/train0": 1.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.2, + "2015956957/train2": 0.2, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 1.0, + "2141802816/train0": 1.0, + "2141802816/train1": 1.0, + "2141802816/train2": 1.0, + "256770874/train0": 1.0, + "256770874/train1": 1.0, + "256770874/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "416183106/train0": 1.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 1.0, + "551504635/train0": 1.0, + "551504635/train1": 1.0, + "551504635/train2": 1.0, + "565369392/train0": 0.6666666666666666, + "565369392/train1": 0.6666666666666666, + "565369392/train2": 0.6666666666666666, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 1.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 1.0, + "75070641/train1": 1.0, + "75070641/train2": 1.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 1.0, + "77147264/train2": 1.0, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 1.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 1.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.0, + "low": 0.5941176470588235, + "mean": 0.8647058823529413, + "n": 5, + "num_samples": 500 + }, + "mean": 0.8647058823529413, + "per_task": { + "LiftPegUpright-v1": 1.0, + "PickCube-v1": 1.0, + "PullCube-v1": 0.3235294117647059, + "PushCube-v1": 1.0, + "StackCube-v1": 1.0 + } + }, + "metric": "hidden_chart_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.7246376811594203, + "low": 0.6086956521739131, + "mean": 0.6666666666666666, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "hidden_chart_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.8005860548316728, + "low": 1.5936215176297697, + "mean": 1.706202811163896, + "n": 141, + "num_samples": 500 + }, + "mean": 1.706202811163896, + "per_seed": { + "1025740212/train0": 2.0, + "1025740212/train1": 2.0, + "1025740212/train2": 2.0, + "1055157354/train0": 2.0, + "1055157354/train1": 2.0, + "1055157354/train2": 2.0, + "1055567618/train0": 2.0, + "1055567618/train1": 2.0, + "1055567618/train2": 2.0, + "1113268940/train0": 2.0, + "1113268940/train1": 2.0, + "1113268940/train2": 2.0, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 2.0, + "1176943004/train1": 2.0, + "1176943004/train2": 2.0, + "1189392917/train0": 2.0, + "1189392917/train1": 2.0, + "1189392917/train2": 2.0, + "1231780997/train0": 2.0, + "1231780997/train1": 2.0, + "1231780997/train2": 2.0, + "1271718007/train0": 2.0, + "1271718007/train1": 2.0, + "1271718007/train2": 2.0, + "1273247072/train0": 2.0, + "1273247072/train1": 2.0, + "1273247072/train2": 2.0, + "1358421325/train0": 2.0, + "1358421325/train1": 2.0, + "1358421325/train2": 2.0, + "1375492786/train0": 0.24345096945762634, + "1375492786/train1": 0.24345096945762634, + "1375492786/train2": 0.24345096945762634, + "141984615/train0": 0.20044784247875214, + "141984615/train1": 0.20044784247875214, + "141984615/train2": 0.20044784247875214, + "1475440973/train0": 2.0, + "1475440973/train1": 2.0, + "1475440973/train2": 2.0, + "1483645336/train0": 2.0, + "1483645336/train1": 2.0, + "1483645336/train2": 2.0, + "1522272686/train0": 2.0, + "1522272686/train1": 2.0, + "1522272686/train2": 2.0, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 2.0, + "1549324805/train0": 1.4027505218982697, + "1549324805/train1": 1.4027505218982697, + "1549324805/train2": 1.4027505218982697, + "1607773860/train0": 2.0, + "1607773860/train1": 2.0, + "1607773860/train2": 2.0, + "1665739511/train0": 2.0, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 2.0, + "1747178030/train2": 2.0, + "1881223807/train0": 2.0, + "1881223807/train1": 2.0, + "1881223807/train2": 2.0, + "1919198188/train0": 2.0, + "1919198188/train1": 2.0, + "1919198188/train2": 2.0, + "1991414747/train0": 2.0, + "1991414747/train1": 2.0, + "1991414747/train2": 2.0, + "1997090716/train0": 2.0, + "1997090716/train1": 2.0, + "1997090716/train2": 2.0, + "2015956957/train0": 0.5298813670873642, + "2015956957/train1": 0.5298813670873642, + "2015956957/train2": 0.5298813670873642, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 2.0, + "211245842/train1": 2.0, + "211245842/train2": 2.0, + "2141802816/train0": 2.0, + "2141802816/train1": 2.0, + "2141802816/train2": 2.0, + "256770874/train0": 2.0, + "256770874/train1": 2.0, + "256770874/train2": 2.0, + "278008524/train0": 1.113811194896698, + "278008524/train1": 1.113811194896698, + "278008524/train2": 1.113811194896698, + "416183106/train0": 2.0, + "416183106/train1": 2.0, + "416183106/train2": 2.0, + "534717141/train0": 2.0, + "534717141/train1": 2.0, + "534717141/train2": 2.0, + "551504635/train0": 2.0, + "551504635/train1": 2.0, + "551504635/train2": 2.0, + "565369392/train0": 1.4266315797964733, + "565369392/train1": 1.4266315797964733, + "565369392/train2": 1.4266315797964733, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.23764699200789133, + "585904130/train1": 0.23764699200789133, + "585904130/train2": 0.23764699200789133, + "626486813/train0": 0.6124684102833271, + "626486813/train1": 0.6124684102833271, + "626486813/train2": 0.6124684102833271, + "636680137/train0": 2.0, + "636680137/train1": 2.0, + "636680137/train2": 2.0, + "706479600/train0": 2.0, + "706479600/train1": 2.0, + "706479600/train2": 2.0, + "750448296/train0": 0.23580759018659592, + "750448296/train1": 0.23580759018659592, + "750448296/train2": 0.23580759018659592, + "75070641/train0": 2.0, + "75070641/train1": 2.0, + "75070641/train2": 2.0, + "767747424/train0": 0.18863565661013126, + "767747424/train1": 0.18863565661013126, + "767747424/train2": 0.18863565661013126, + "77147264/train0": 2.0, + "77147264/train1": 2.0, + "77147264/train2": 2.0, + "794181626/train0": 2.0, + "794181626/train1": 2.0, + "794181626/train2": 2.0, + "957101489/train0": 2.0, + "957101489/train1": 2.0, + "957101489/train2": 2.0, + "998761162/train0": 2.0, + "998761162/train1": 2.0, + "998761162/train2": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 2.0, + "low": 1.2693931020796299, + "mean": 1.7564643673598765, + "n": 5, + "num_samples": 500 + }, + "mean": 1.7564643673598765, + "per_task": { + "LiftPegUpright-v1": 2.0, + "PickCube-v1": 2.0, + "PullCube-v1": 0.7823218367993832, + "PushCube-v1": 2.0, + "StackCube-v1": 2.0 + } + }, + "metric": "hidden_chart_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.5056910838409898, + "low": 1.293966528766109, + "mean": 1.3999846732054932, + "n": 207, + "num_samples": 500 }, "n": 207 }, @@ -197,11 +948,11 @@ "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5674940898345152, - "low": 0.4113475177304965, + "high": 0.5641843971631206, + "low": 0.4092198581560284, "mean": 0.4849881796690307, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.4849881796690307, "per_seed": { @@ -352,10 +1103,10 @@ "ci": { "confidence": 0.95, "high": 0.6328205128205128, - "low": 0.4294117647058823, + "low": 0.42745098039215684, "mean": 0.5411161387631975, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.5411161387631975, "per_task": { @@ -369,11 +1120,11 @@ "metric": "outcome_ptr_at_8", "micro": { "confidence": 0.95, - "high": 0.5265700483091788, + "high": 0.5314009661835749, "low": 0.391304347826087, "mean": 0.45893719806763283, "n": 207, - "num_samples": 1000 + "num_samples": 500 }, "n": 207 }, @@ -381,11 +1132,11 @@ "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.17446188421793687, - "low": 0.14114248449490324, + "high": 0.1737242314731193, + "low": 0.1410392771878472, "mean": 0.15793930902529071, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.15793930902529071, "per_seed": { @@ -535,11 +1286,11 @@ "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.17042242389065887, - "low": 0.1574042611063619, + "high": 0.17033533868825532, + "low": 0.15683265673150842, "mean": 0.16359339232485098, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.16359339232485098, "per_task": { @@ -553,195 +1304,2219 @@ "metric": "pairwise_causal_calibration_ece", "micro": { "confidence": 0.95, - "high": 0.1771431506285629, - "low": 0.14633605302901057, + "high": 0.1771397879569765, + "low": 0.14678388743440304, "mean": 0.16176667684993604, "n": 207, - "num_samples": 1000 + "num_samples": 500 }, "n": 207 }, - "selector_regret_at_8": { + "proposal_oracle_success_at_8": { "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.5194647902708409, - "low": 0.2999603762294949, - "mean": 0.40561739670197494, + "high": 0.5497635933806146, + "low": 0.38581560283687943, + "mean": 0.47565011820330966, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, - "mean": 0.40561739670197494, + "mean": 0.47565011820330966, "per_seed": { - "1025740212/train0": 0.26188503205776215, - "1025740212/train1": 0.47739143669605255, - "1025740212/train2": 1.772506669163704, - "1055157354/train0": 1.8516457676887512, - "1055157354/train1": 0.6370042562484741, - "1055157354/train2": 0.03480546176433563, - "1055567618/train0": 1.183156132698059, - "1055567618/train1": 1.8253512382507324, - "1055567618/train2": 1.9218642711639404, - "1113268940/train0": 0.38470593094825745, - "1113268940/train1": 0.18306860327720642, - "1113268940/train2": 0.06608206033706665, - "1113371261/train0": 0.0, - "1113371261/train1": 0.0, - "1113371261/train2": 0.0, - "1176943004/train0": 0.0, - "1176943004/train1": 1.974869817495346, - "1176943004/train2": 0.0, - "1189392917/train0": 0.0, - "1189392917/train1": 0.0, - "1189392917/train2": 0.0, - "1231780997/train0": 0.4897426515817642, - "1231780997/train1": 0.0, - "1231780997/train2": 0.5006794333457947, - "1271718007/train0": 0.0, - "1271718007/train1": 0.00353144109249115, - "1271718007/train2": 0.19734752923250198, + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 1.0, + "1176943004/train1": 1.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 1.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, "1273247072/train0": 0.0, - "1273247072/train1": 0.45726464316248894, - "1273247072/train2": 0.35906313359737396, - "1358421325/train0": 0.5775325745344162, - "1358421325/train1": 0.6467066705226898, - "1358421325/train2": 1.90096914768219, - "1375492786/train0": 0.0002854267368093133, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 1.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 1.0, + "1607773860/train1": 0.0, + "1607773860/train2": 1.0, + "1665739511/train0": 0.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 1.0, + "1991414747/train1": 1.0, + "1991414747/train2": 1.0, + "1997090716/train0": 1.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 1.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 1.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 1.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.0, + "565369392/train2": 0.0, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "706479600/train0": 1.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 0.5, + "77147264/train2": 1.0, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 0.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5987933634992458, + "low": 0.14901960784313725, + "mean": 0.3799899446958271, + "n": 5, + "num_samples": 500 + }, + "mean": 0.3799899446958271, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 0.0, + "PullCube-v1": 0.18627450980392157, + "PushCube-v1": 0.7692307692307693, + "StackCube-v1": 0.4444444444444444 + } + }, + "metric": "proposal_oracle_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.4444444444444444, + "low": 0.30917874396135264, + "mean": 0.37681159420289856, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "proposal_oracle_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2335697399527187, + "low": 0.06122931442080378, + "mean": 0.15295508274231676, + "n": 141, + "num_samples": 500 + }, + "mean": 0.15295508274231676, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.0, + "1176943004/train1": 1.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.5, + "1271718007/train1": 0.0, + "1271718007/train2": 0.0, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 1.0, + "1375492786/train0": -1.0, + "1375492786/train1": -1.0, + "1375492786/train2": -1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 1.0, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": -0.3333333333333333, + "1549324805/train2": -0.3333333333333333, + "1607773860/train0": 0.0, + "1607773860/train1": -1.0, + "1607773860/train2": 0.0, + "1665739511/train0": -1.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 0.0, + "1991414747/train1": 0.0, + "1991414747/train2": 0.0, + "1997090716/train0": 1.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 1.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 1.0, + "565369392/train0": 0.0, + "565369392/train1": -0.3333333333333333, + "565369392/train2": -0.3333333333333333, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": -0.25, + "626486813/train2": -0.25, + "636680137/train0": -1.0, + "636680137/train1": -1.0, + "636680137/train2": -1.0, + "706479600/train0": 1.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": -0.25, + "767747424/train2": -0.25, + "77147264/train0": 0.5, + "77147264/train1": 0.0, + "77147264/train2": 0.5, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 0.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.3107692307692308, + "low": -0.01581699346405229, + "mean": 0.1427048768225239, + "n": 5, + "num_samples": 500 + }, + "mean": 0.1427048768225239, + "per_task": { + "LiftPegUpright-v1": 0.16666666666666666, + "PickCube-v1": 0.0, + "PullCube-v1": -0.10784313725490197, + "PushCube-v1": 0.41025641025641024, + "StackCube-v1": 0.24444444444444444 + } + }, + "metric": "proposal_oracle_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.15942028985507245, + "low": 0.01932367149758454, + "mean": 0.09178743961352658, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "proposal_oracle_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.2280026059425997, + "low": 0.9486234303862167, + "mean": 1.1034544456804372, + "n": 141, + "num_samples": 500 + }, + "mean": 1.1034544456804372, + "per_seed": { + "1025740212/train0": 0.4761262834072113, + "1025740212/train1": 0.6983401775360107, + "1025740212/train2": 2.0, + "1055157354/train0": 2.0, + "1055157354/train1": 0.8232162594795227, + "1055157354/train2": 0.22467364370822906, + "1055567618/train0": 2.0, + "1055567618/train1": 2.0, + "1055567618/train2": 2.0, + "1113268940/train0": 0.6646121144294739, + "1113268940/train1": 0.6617497801780701, + "1113268940/train2": 0.7025249004364014, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 2.0, + "1176943004/train1": 2.0, + "1176943004/train2": 2.0, + "1189392917/train0": 2.0, + "1189392917/train1": 2.0, + "1189392917/train2": 2.0, + "1231780997/train0": 0.7187411189079285, + "1231780997/train1": 2.0, + "1231780997/train2": 0.7278411984443665, + "1271718007/train0": 2.0, + "1271718007/train1": 1.1110476776957512, + "1271718007/train2": 1.2635672688484192, + "1273247072/train0": 0.13278456032276154, + "1273247072/train1": 0.4997493326663971, + "1273247072/train2": 0.46377021074295044, + "1358421325/train0": 0.8142459392547607, + "1358421325/train1": 0.8275434374809265, + "1358421325/train2": 2.0, + "1375492786/train0": 0.0016440550098195672, + "1375492786/train1": 0.0017900466918945312, + "1375492786/train2": 0.0008872548933140934, + "141984615/train0": 0.0797463705142339, + "141984615/train1": 0.09240151196718216, + "141984615/train2": 0.054234166940053306, + "1475440973/train0": 2.0, + "1475440973/train1": 2.0, + "1475440973/train2": 2.0, + "1483645336/train0": 2.0, + "1483645336/train1": 2.0, + "1483645336/train2": 2.0, + "1522272686/train0": 0.2435554713010788, + "1522272686/train1": 0.23248937726020813, + "1522272686/train2": 0.1949000507593155, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 2.0, + "1549324805/train0": 1.3427665165315072, + "1549324805/train1": 0.6785067518552145, + "1549324805/train2": 0.6833462389186025, + "1607773860/train0": 2.0, + "1607773860/train1": 0.8267073631286621, + "1607773860/train2": 2.0, + "1665739511/train0": 0.8216220736503601, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 0.11134227365255356, + "1747178030/train2": 0.12480536848306656, + "1881223807/train0": 0.0513521283864975, + "1881223807/train1": 0.034886181354522705, + "1881223807/train2": 0.043482705950737, + "1919198188/train0": 0.08992795646190643, + "1919198188/train1": 0.046244338154792786, + "1919198188/train2": 0.06275103986263275, + "1991414747/train0": 2.0, + "1991414747/train1": 2.0, + "1991414747/train2": 2.0, + "1997090716/train0": 2.0, + "1997090716/train1": 2.0, + "1997090716/train2": 2.0, + "2015956957/train0": 0.45124312303960323, + "2015956957/train1": 0.05629209652543068, + "2015956957/train2": 0.43981083631515505, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 0.8130142092704773, + "211245842/train1": 0.8358442187309265, + "211245842/train2": 2.0, + "2141802816/train0": 0.12472955882549286, + "2141802816/train1": 0.09589323401451111, + "2141802816/train2": 2.0, + "256770874/train0": 0.24107472598552704, + "256770874/train1": 0.8125, + "256770874/train2": 0.21061499416828156, + "278008524/train0": 0.011432985425926745, + "278008524/train1": 0.03147180867381394, + "278008524/train2": 0.032865648390725255, + "416183106/train0": 0.5778142213821411, + "416183106/train1": 2.0, + "416183106/train2": 2.0, + "534717141/train0": 2.0, + "534717141/train1": 2.0, + "534717141/train2": 2.0, + "551504635/train0": 0.7250897884368896, + "551504635/train1": 0.7224790453910828, + "551504635/train2": 2.0, + "565369392/train0": 0.7852714310089747, + "565369392/train1": 0.17366480827331543, + "565369392/train2": 0.1326704372962316, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.05598785417775313, + "585904130/train1": 0.032918572425842285, + "585904130/train2": 0.07848061621189117, + "626486813/train0": 0.5615248780231923, + "626486813/train1": 0.07365635968744755, + "626486813/train2": 0.07003506645560265, + "636680137/train0": 0.012414653785526752, + "636680137/train1": 0.013928472995758057, + "636680137/train2": 0.02436021901667118, + "706479600/train0": 2.0, + "706479600/train1": 2.0, + "706479600/train2": 2.0, + "750448296/train0": 0.1069931760430336, + "750448296/train1": 0.07342664152383804, + "750448296/train2": 0.0669428389519453, + "75070641/train0": 0.2396746575832367, + "75070641/train1": 0.8128874897956848, + "75070641/train2": 0.5650836229324341, + "767747424/train0": 0.5703969299793243, + "767747424/train1": 0.11342880013398826, + "767747424/train2": 0.11239026742987335, + "77147264/train0": 2.0, + "77147264/train1": 1.426636129617691, + "77147264/train2": 2.0, + "794181626/train0": 2.0, + "794181626/train1": 2.0, + "794181626/train2": 2.0, + "957101489/train0": 0.5881698727607727, + "957101489/train1": 2.0, + "957101489/train2": 2.0, + "998761162/train0": 0.08403940498828888, + "998761162/train1": 2.0, + "998761162/train2": 2.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.3557187332811518, + "low": 0.552532447482158, + "mean": 0.971318265720506, + "n": 5, + "num_samples": 500 + }, + "mean": 0.971318265720506, + "per_task": { + "LiftPegUpright-v1": 1.3387502448426352, + "PickCube-v1": 0.3654347012440364, + "PullCube-v1": 0.4324846717342963, + "PushCube-v1": 1.5530982193274376, + "StackCube-v1": 1.1668234914541245 + } + }, + "metric": "proposal_oracle_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 1.0042415133076787, + "low": 0.758569956281625, + "mean": 0.8810875975700648, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "proposal_oracle_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.40175874303166603, + "low": 0.10017959579071863, + "mean": 0.2610664251762173, + "n": 141, + "num_samples": 500 + }, + "mean": 0.2610664251762173, + "per_seed": { + "1025740212/train0": -0.05224737524986267, + "1025740212/train1": 0.16996651887893677, + "1025740212/train2": 1.471626341342926, + "1055157354/train0": 1.8235089033842087, + "1055157354/train1": 0.6467251628637314, + "1055157354/train2": 0.02818956971168518, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 0.24229958653450012, + "1113268940/train1": 0.2394372522830963, + "1113268940/train2": 0.2802123725414276, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.949059471487999, + "1176943004/train1": 1.9419668167829514, + "1176943004/train2": 1.949059471487999, + "1189392917/train0": 1.8915813565254211, + "1189392917/train1": 1.8915813565254211, + "1189392917/train2": 1.8915813565254211, + "1231780997/train0": -0.008265018463134766, + "1231780997/train1": 1.2729938626289368, + "1231780997/train2": 0.0008350610733032227, + "1271718007/train0": 0.5857395231723785, + "1271718007/train1": -0.3043334558606148, + "1271718007/train2": -0.15181386470794678, + "1273247072/train0": -0.02064509689807892, + "1273247072/train1": 0.34631967544555664, + "1273247072/train2": 0.31034055352211, + "1358421325/train0": -0.018456339836120605, + "1358421325/train1": -0.004933059215545654, + "1358421325/train2": 1.1672977209091187, + "1375492786/train0": -1.9983559449901804, + "1375492786/train1": -1.9982099533081055, + "1375492786/train2": -1.999112745106686, + "141984615/train0": -0.01406576856970787, + "141984615/train1": -0.001410627116759618, + "141984615/train2": -0.039577972143888474, + "1475440973/train0": 0.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 1.9835839420557022, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 0.048794373869895935, + "1522272686/train1": 0.03404626250267029, + "1522272686/train2": 0.0001389533281326294, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 0.0, + "1549324805/train0": -0.035980931793649994, + "1549324805/train1": -0.7002406964699427, + "1549324805/train2": -0.6954012094065547, + "1607773860/train0": 0.0, + "1607773860/train1": -1.173292636871338, + "1607773860/train2": 0.0, + "1665739511/train0": -1.17837792634964, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.8536934107542038, + "1747178030/train1": -0.034964315593242645, + "1747178030/train2": -0.021501220762729645, + "1881223807/train0": 0.0193251371383667, + "1881223807/train1": 0.0028591901063919067, + "1881223807/train2": 0.011455714702606201, + "1919198188/train0": 0.05272909998893738, + "1919198188/train1": 0.009045466780662537, + "1919198188/train2": 0.025552168488502502, + "1991414747/train0": 0.0, + "1991414747/train1": 0.0, + "1991414747/train2": 0.0, + "1997090716/train0": 1.8607252091169357, + "1997090716/train1": 1.8607252091169357, + "1997090716/train2": 1.8607252091169357, + "2015956957/train0": 0.38412333317101, + "2015956957/train1": -0.010827693343162536, + "2015956957/train2": 0.3726910464465618, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": -0.02582019567489624, + "211245842/train1": -0.0029901862144470215, + "211245842/train2": 1.1611655950546265, + "2141802816/train0": 0.05392231047153473, + "2141802816/train1": 0.02508598566055298, + "2141802816/train2": 1.9291927516460419, + "256770874/train0": -0.6146733313798904, + "256770874/train1": -0.04324805736541748, + "256770874/train2": -0.6451330631971359, + "278008524/train0": -0.09603888296987861, + "278008524/train1": -0.07600005972199142, + "278008524/train2": -0.0746062200050801, + "416183106/train0": 0.37748780846595764, + "416183106/train1": 1.7996735870838165, + "416183106/train2": 1.7996735870838165, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0011172294616699219, + "551504635/train1": -0.00224381685256958, + "551504635/train2": 1.276113212108612, + "565369392/train0": -0.014494215448697409, + "565369392/train1": -0.6261008381843567, + "565369392/train2": -0.6670952091614405, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": -0.019304243847727776, + "585904130/train1": -0.04237352559963862, + "585904130/train2": 0.0031885181864102683, + "626486813/train0": 0.0560848613968119, + "626486813/train1": -0.43178365693893284, + "626486813/train2": -0.43540495017077774, + "636680137/train0": -1.9875853462144732, + "636680137/train1": -1.986071527004242, + "636680137/train2": -1.9756397809833288, + "706479600/train0": 1.1874593496322632, + "706479600/train1": 1.1649112105369568, + "706479600/train2": 1.1649112105369568, + "750448296/train0": -0.004540383815765381, + "750448296/train1": -0.03810691833496094, + "750448296/train2": -0.044590720906853676, + "75070641/train0": 0.03692302107810974, + "75070641/train1": 0.6101358532905579, + "75070641/train2": 0.36233198642730713, + "767747424/train0": 0.022872742265462875, + "767747424/train1": -0.4340953875798732, + "767747424/train2": -0.4351339202839881, + "77147264/train0": 0.5784205198287964, + "77147264/train1": 0.005056649446487427, + "77147264/train2": 0.5784205198287964, + "794181626/train0": 1.826662853360176, + "794181626/train1": 1.8190439492464066, + "794181626/train2": 1.8190439492464066, + "957101489/train0": 0.3842049241065979, + "957101489/train1": 1.7962781339883804, + "957101489/train2": 1.7960350513458252, + "998761162/train0": 0.015134349465370178, + "998761162/train1": 1.9321834295988083, + "998761162/train2": 1.9321834295988083 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5660580151865624, + "low": -0.01856180906115269, + "mean": 0.28212140274398645, + "n": 5, + "num_samples": 500 + }, + "mean": 0.28212140274398645, + "per_task": { + "LiftPegUpright-v1": 0.2717691792382134, + "PickCube-v1": 0.2120050440231959, + "PullCube-v1": -0.21874524904905782, + "PushCube-v1": 0.7821719508904678, + "StackCube-v1": 0.3634060886171129 + } + }, + "metric": "proposal_oracle_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.2666971658560074, + "low": 0.03155638067167621, + "mean": 0.14528417887851136, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_success_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.3537825059101655, + "low": 0.21040189125295505, + "mean": 0.284160756501182, + "n": 141, + "num_samples": 500 + }, + "mean": 0.284160756501182, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 1.0, + "1113371261/train1": 1.0, + "1113371261/train2": 1.0, + "1176943004/train0": 1.0, + "1176943004/train1": 0.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 1.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 1.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 1.0, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 1.0, + "1528740417/train1": 1.0, + "1528740417/train2": 0.0, + "1549324805/train0": 0.6666666666666666, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 0.0, + "1607773860/train1": 0.0, + "1607773860/train2": 1.0, + "1665739511/train0": 0.0, + "1665739511/train1": 1.0, + "1665739511/train2": 1.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 1.0, + "1991414747/train1": 0.0, + "1991414747/train2": 1.0, + "1997090716/train0": 0.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 1.0, + "2054899126/train1": 1.0, + "2054899126/train2": 1.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 0.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 1.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 0.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.0, + "565369392/train2": 0.0, + "565621826/train0": 1.0, + "565621826/train1": 1.0, + "565621826/train2": 1.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.25, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "706479600/train0": 0.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.25, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 0.0, + "77147264/train1": 0.5, + "77147264/train2": 1.0, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 0.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.39418803418803416, + "low": 0.07777777777777777, + "mean": 0.22102564102564098, + "n": 5, + "num_samples": 500 + }, + "mean": 0.22102564102564098, + "per_task": { + "LiftPegUpright-v1": 0.2222222222222222, + "PickCube-v1": 0.0, + "PullCube-v1": 0.16666666666666666, + "PushCube-v1": 0.5384615384615384, + "StackCube-v1": 0.17777777777777778 + } + }, + "metric": "selected_success_at_8", + "micro": { + "confidence": 0.95, + "high": 0.30434782608695654, + "low": 0.18840579710144928, + "mean": 0.24154589371980675, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_success_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.05035460992907801, + "low": -0.13687943262411348, + "mean": -0.038534278959810875, + "n": 141, + "num_samples": 500 + }, + "mean": -0.038534278959810875, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": -1.0, + "1055567618/train1": -1.0, + "1055567618/train2": -1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.0, + "1176943004/train1": 0.0, + "1176943004/train2": 1.0, + "1189392917/train0": 1.0, + "1189392917/train1": 1.0, + "1189392917/train2": 1.0, + "1231780997/train0": 0.0, + "1231780997/train1": 1.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.5, + "1271718007/train1": 0.0, + "1271718007/train2": 0.0, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 0.0, + "1375492786/train0": -1.0, + "1375492786/train1": -1.0, + "1375492786/train2": -1.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": -1.0, + "1475440973/train2": -1.0, + "1483645336/train0": 1.0, + "1483645336/train1": -1.0, + "1483645336/train2": -1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": -1.0, + "1549324805/train0": 0.0, + "1549324805/train1": -0.3333333333333333, + "1549324805/train2": -0.3333333333333333, + "1607773860/train0": -1.0, + "1607773860/train1": -1.0, + "1607773860/train2": 0.0, + "1665739511/train0": -1.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 0.0, + "1991414747/train1": -1.0, + "1991414747/train2": 0.0, + "1997090716/train0": 0.0, + "1997090716/train1": 1.0, + "1997090716/train2": 1.0, + "2015956957/train0": 0.2, + "2015956957/train1": 0.0, + "2015956957/train2": 0.2, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 1.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 0.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": -1.0, + "534717141/train1": -1.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 0.0, + "565369392/train0": 0.0, + "565369392/train1": -0.3333333333333333, + "565369392/train2": -0.3333333333333333, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": -0.25, + "626486813/train2": -0.25, + "636680137/train0": -1.0, + "636680137/train1": -1.0, + "636680137/train2": -1.0, + "706479600/train0": 0.0, + "706479600/train1": 1.0, + "706479600/train2": 1.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": -0.25, + "767747424/train2": -0.25, + "77147264/train0": -0.5, + "77147264/train1": 0.0, + "77147264/train2": 0.5, + "794181626/train0": 1.0, + "794181626/train1": 1.0, + "794181626/train2": 1.0, + "957101489/train0": 0.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 0.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.08102564102564103, + "low": -0.09986928104575163, + "mean": -0.01625942684766214, + "n": 5, + "num_samples": 500 + }, + "mean": -0.01625942684766214, + "per_task": { + "LiftPegUpright-v1": -0.1111111111111111, + "PickCube-v1": 0.0, + "PullCube-v1": -0.12745098039215685, + "PushCube-v1": 0.1794871794871795, + "StackCube-v1": -0.022222222222222223 + } + }, + "metric": "selected_success_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": 0.024154589371980676, + "low": -0.1111111111111111, + "mean": -0.043478260869565216, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_utility_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.8244129490417189, + "low": 0.5588968922524719, + "mean": 0.6978370489784622, + "n": 141, + "num_samples": 500 + }, + "mean": 0.6978370489784622, + "per_seed": { + "1025740212/train0": 0.21424125134944916, + "1025740212/train1": 0.2209487408399582, + "1025740212/train2": 0.22749333083629608, + "1055157354/train0": 0.14835423231124878, + "1055157354/train1": 0.18621200323104858, + "1055157354/train2": 0.18986818194389343, + "1055567618/train0": 0.8168438673019409, + "1055567618/train1": 0.17464876174926758, + "1055567618/train2": 0.07813572883605957, + "1113268940/train0": 0.27990618348121643, + "1113268940/train1": 0.47868117690086365, + "1113268940/train2": 0.6364428400993347, + "1113371261/train0": 2.0, + "1113371261/train1": 2.0, + "1113371261/train2": 2.0, + "1176943004/train0": 2.0, + "1176943004/train1": 0.02513018250465393, + "1176943004/train2": 2.0, + "1189392917/train0": 2.0, + "1189392917/train1": 2.0, + "1189392917/train2": 2.0, + "1231780997/train0": 0.22899846732616425, + "1231780997/train1": 2.0, + "1231780997/train2": 0.22716176509857178, + "1271718007/train0": 2.0, + "1271718007/train1": 1.10751623660326, + "1271718007/train2": 1.0662197396159172, + "1273247072/train0": 0.13278456032276154, + "1273247072/train1": 0.04248468950390816, + "1273247072/train2": 0.10470707714557648, + "1358421325/train0": 0.23671336472034454, + "1358421325/train1": 0.1808367669582367, + "1358421325/train2": 0.09903085231781006, + "1375492786/train0": 0.001358628273010254, + "1375492786/train1": 0.001317779184319079, + "1375492786/train2": 0.0005992253427393734, + "141984615/train0": 0.04657847310105959, + "141984615/train1": 0.03509971871972084, + "141984615/train2": 0.04040361692508062, + "1475440973/train0": 2.0, + "1475440973/train1": 0.710078775882721, + "1475440973/train2": 0.7161068916320801, + "1483645336/train0": 2.0, + "1483645336/train1": 0.06921124458312988, + "1483645336/train2": 0.0313941091299057, + "1522272686/train0": 0.2340310961008072, + "1522272686/train1": 0.19632326066493988, + "1522272686/train2": 0.17692717909812927, + "1528740417/train0": 2.0, + "1528740417/train1": 2.0, + "1528740417/train2": 0.7195272445678711, + "1549324805/train0": 1.3393354682872693, + "1549324805/train1": 0.6769271823577583, + "1549324805/train2": 0.6709667841593424, + "1607773860/train0": 0.220267653465271, + "1607773860/train1": 0.2231033444404602, + "1607773860/train2": 2.0, + "1665739511/train0": 0.8140708208084106, + "1665739511/train1": 2.0, + "1665739511/train2": 2.0, + "1747178030/train0": 2.0, + "1747178030/train1": 0.08418524265289307, + "1747178030/train2": 0.09071242809295654, + "1881223807/train0": 0.03111603856086731, + "1881223807/train1": 0.025857940316200256, + "1881223807/train2": 0.018627971410751343, + "1919198188/train0": 0.07307347655296326, + "1919198188/train1": 0.046244338154792786, + "1919198188/train2": 0.043911516666412354, + "1991414747/train0": 2.0, + "1991414747/train1": 0.013612315058708191, + "1991414747/train2": 2.0, + "1997090716/train0": 0.03083018958568573, + "1997090716/train1": 2.0, + "1997090716/train2": 2.0, + "2015956957/train0": 0.433539473824203, + "2015956957/train1": 0.034096240811049935, + "2015956957/train2": 0.4342371065169573, + "2054899126/train0": 2.0, + "2054899126/train1": 2.0, + "2054899126/train2": 2.0, + "211245842/train0": 0.22217948734760284, + "211245842/train1": 0.15655378997325897, + "211245842/train2": 2.0, + "2141802816/train0": 0.0492904931306839, + "2141802816/train1": 0.07612672448158264, + "2141802816/train2": 0.13836362957954407, + "256770874/train0": 0.13286492228507996, + "256770874/train1": 0.10803383588790894, + "256770874/train2": 0.16439804434776306, + "278008524/train0": 0.009787360846530646, + "278008524/train1": 0.020062235475052148, + "278008524/train2": 0.012664265930652618, + "416183106/train0": 0.24176457524299622, + "416183106/train1": 0.23312881588935852, + "416183106/train2": 0.18704846501350403, + "534717141/train0": 0.07250408083200455, + "534717141/train1": 0.04410672187805176, + "534717141/train2": 2.0, + "551504635/train0": 0.6918362975120544, + "551504635/train1": 0.7224790453910828, + "551504635/train2": 0.726307213306427, + "565369392/train0": 0.7208344464500746, + "565369392/train1": 0.12740397453308105, + "565369392/train2": 0.08709763859709103, + "565621826/train0": 2.0, + "565621826/train1": 2.0, + "565621826/train2": 2.0, + "585904130/train0": 0.012818952556699514, + "585904130/train1": 0.02085024770349264, + "585904130/train2": 0.06744805444031954, + "626486813/train0": 0.5429544361541048, + "626486813/train1": 0.04864858591463417, + "626486813/train2": 0.03831234632525593, + "636680137/train0": 0.012414653785526752, + "636680137/train1": 0.013928472995758057, + "636680137/train2": 0.017868518829345703, + "706479600/train0": 0.8134256601333618, + "706479600/train1": 2.0, + "706479600/train2": 2.0, + "750448296/train0": 0.029670913703739643, + "750448296/train1": 0.046416282653808594, + "750448296/train2": 0.049234201200306416, + "75070641/train0": 0.16485285758972168, + "75070641/train1": 0.22657974064350128, + "75070641/train2": 0.194048672914505, + "767747424/train0": 0.554436425678432, + "767747424/train1": 0.06781683932058513, + "767747424/train2": 0.0877318112179637, + "77147264/train0": 0.13140545785427094, + "77147264/train1": 1.426636129617691, + "77147264/train2": 2.0, + "794181626/train0": 2.0, + "794181626/train1": 2.0, + "794181626/train2": 2.0, + "957101489/train0": 0.5881698727607727, + "957101489/train1": 0.22424602508544922, + "957101489/train2": 0.567138671875, + "998761162/train0": 0.006543517112731934, + "998761162/train1": 0.04874563217163086, + "998761162/train2": 0.0608120858669281 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.886751612873159, + "low": 0.2852983943786886, + "mean": 0.5914274646835725, + "n": 5, + "num_samples": 500 + }, + "mean": 0.5914274646835725, + "per_task": { + "LiftPegUpright-v1": 0.8222338457902273, + "PickCube-v1": 0.09332544232408206, + "PullCube-v1": 0.372572739694756, + "PushCube-v1": 1.0957474731481993, + "StackCube-v1": 0.5732578224605984 + } + }, + "metric": "selected_utility_at_8", + "micro": { + "confidence": 0.95, + "high": 0.7023529558462779, + "low": 0.48535686017352647, + "mean": 0.5875041471669958, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selected_utility_gain_over_base_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.01412016182106394, + "low": -0.30047688159442687, + "mean": -0.14455097152575763, + "n": 141, + "num_samples": 500 + }, + "mean": -0.14455097152575763, + "per_seed": { + "1025740212/train0": -0.3141324073076248, + "1025740212/train1": -0.3074249178171158, + "1025740212/train2": -0.3008803278207779, + "1055157354/train0": -0.02813686430454254, + "1055157354/train1": 0.009720906615257263, + "1055157354/train2": -0.006615892052650452, + "1055567618/train0": -1.183156132698059, + "1055567618/train1": -1.8253512382507324, + "1055567618/train2": -1.9218642711639404, + "1113268940/train0": -0.14240634441375732, + "1113268940/train1": 0.05636864900588989, + "1113268940/train2": 0.21413031220436096, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 1.949059471487999, + "1176943004/train1": -0.032903000712394714, + "1176943004/train2": 1.949059471487999, + "1189392917/train0": 1.8915813565254211, + "1189392917/train1": 1.8915813565254211, + "1189392917/train2": 1.8915813565254211, + "1231780997/train0": -0.498007670044899, + "1231780997/train1": 1.2729938626289368, + "1231780997/train2": -0.49984437227249146, + "1271718007/train0": 0.5857395231723785, + "1271718007/train1": -0.3078648969531059, + "1271718007/train2": -0.34916139394044876, + "1273247072/train0": -0.02064509689807892, + "1273247072/train1": -0.1109449677169323, + "1273247072/train2": -0.04872258007526398, + "1358421325/train0": -0.5959889143705368, + "1358421325/train1": -0.6516397297382355, + "1358421325/train2": -0.7336714267730713, + "1375492786/train0": -1.9986413717269897, + "1375492786/train1": -1.998682220815681, + "1375492786/train2": -1.9994007746572606, + "141984615/train0": -0.04723366598288218, + "141984615/train1": -0.05871242036422094, + "141984615/train2": -0.05340852215886116, + "1475440973/train0": 0.0, + "1475440973/train1": -1.289921224117279, + "1475440973/train2": -1.28389310836792, + "1483645336/train0": 1.9835839420557022, + "1483645336/train1": -1.9307887554168701, + "1483645336/train2": -1.9686058908700943, + "1522272686/train0": 0.03926999866962433, + "1522272686/train1": -0.0021198540925979614, + "1522272686/train2": -0.01783391833305359, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": -1.280472755432129, + "1549324805/train0": -0.03941198003788789, + "1549324805/train1": -0.7018202659673989, + "1549324805/train2": -0.7077806641658148, + "1607773860/train0": -1.779732346534729, + "1607773860/train1": -1.7768966555595398, + "1607773860/train2": 0.0, + "1665739511/train0": -1.1859291791915894, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 1.8536934107542038, + "1747178030/train1": -0.06212134659290314, + "1747178030/train2": -0.05559416115283966, + "1881223807/train0": -0.0009109526872634888, + "1881223807/train1": -0.006169050931930542, + "1881223807/train2": -0.013399019837379456, + "1919198188/train0": 0.0358746200799942, + "1919198188/train1": 0.009045466780662537, + "1919198188/train2": 0.0067126452922821045, + "1991414747/train0": 0.0, + "1991414747/train1": -1.9863876849412918, + "1991414747/train2": 0.0, + "1997090716/train0": -0.10844460129737854, + "1997090716/train1": 1.8607252091169357, + "1997090716/train2": 1.8607252091169357, + "2015956957/train0": 0.3664196839556098, + "2015956957/train1": -0.033023549057543275, + "2015956957/train2": 0.36711731664836406, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": -0.6166549175977707, + "211245842/train1": -0.6822806149721146, + "211245842/train2": 1.1611655950546265, + "2141802816/train0": -0.02151675522327423, + "2141802816/train1": 0.005319476127624512, + "2141802816/train2": 0.06755638122558594, + "256770874/train0": -0.7228831350803375, + "256770874/train1": -0.7477142214775085, + "256770874/train2": -0.6913500130176544, + "278008524/train0": -0.09768450754927471, + "278008524/train1": -0.08740963292075321, + "278008524/train2": -0.09480760246515274, + "416183106/train0": 0.041438162326812744, + "416183106/train1": 0.03280240297317505, + "416183106/train2": -0.013277947902679443, + "534717141/train0": -1.9274959191679955, + "534717141/train1": -1.9558932781219482, + "534717141/train2": 0.0, + "551504635/train0": -0.03213626146316528, + "551504635/train1": -0.00224381685256958, + "551504635/train2": 0.0024204254150390625, + "565369392/train0": -0.07893120000759761, + "565369392/train1": -0.6723616719245911, + "565369392/train2": -0.712668007860581, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": -0.06247314546878139, + "585904130/train1": -0.05444185032198826, + "585904130/train2": -0.007844043585161367, + "626486813/train0": 0.037514419527724385, + "626486813/train1": -0.4567914307117462, + "626486813/train2": -0.46712767030112445, + "636680137/train0": -1.9875853462144732, + "636680137/train1": -1.986071527004242, + "636680137/train2": -1.9821314811706543, + "706479600/train0": 0.000885009765625, + "706479600/train1": 1.1649112105369568, + "706479600/train2": 1.1649112105369568, + "750448296/train0": -0.08186264615505934, + "750448296/train1": -0.06511727720499039, + "750448296/train2": -0.062299358658492565, + "75070641/train0": -0.03789877891540527, + "75070641/train1": 0.02382810413837433, + "75070641/train2": -0.008702963590621948, + "767747424/train0": 0.006912237964570522, + "767747424/train1": -0.47970734839327633, + "767747424/train2": -0.45979237649589777, + "77147264/train0": -1.2901740223169327, + "77147264/train1": 0.005056649446487427, + "77147264/train2": 0.5784205198287964, + "794181626/train0": 1.826662853360176, + "794181626/train1": 1.8190439492464066, + "794181626/train2": 1.8190439492464066, + "957101489/train0": 0.3842049241065979, + "957101489/train1": 0.02052415907382965, + "957101489/train2": 0.3631737232208252, + "998761162/train0": -0.06236153841018677, + "998761162/train1": -0.019070938229560852, + "998761162/train2": -0.007004484534263611 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.09991136278861609, + "low": -0.2553936764363997, + "mean": -0.09776939829294695, + "n": 5, + "num_samples": 500 + }, + "mean": -0.09776939829294695, + "per_task": { + "LiftPegUpright-v1": -0.24474721981419456, + "PickCube-v1": -0.0601042148967584, + "PullCube-v1": -0.2786571810885981, + "PushCube-v1": 0.3248212047112294, + "StackCube-v1": -0.23015958037641313 + } + }, + "metric": "selected_utility_gain_over_base_at_8", + "micro": { + "confidence": 0.95, + "high": -0.02365911047419776, + "low": -0.26539749033037113, + "mean": -0.14829927152455763, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "selector_regret_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.5224681463797662, + "low": 0.2913920145155972, + "mean": 0.40561739670197494, + "n": 141, + "num_samples": 500 + }, + "mean": 0.40561739670197494, + "per_seed": { + "1025740212/train0": 0.26188503205776215, + "1025740212/train1": 0.47739143669605255, + "1025740212/train2": 1.772506669163704, + "1055157354/train0": 1.8516457676887512, + "1055157354/train1": 0.6370042562484741, + "1055157354/train2": 0.03480546176433563, + "1055567618/train0": 1.183156132698059, + "1055567618/train1": 1.8253512382507324, + "1055567618/train2": 1.9218642711639404, + "1113268940/train0": 0.38470593094825745, + "1113268940/train1": 0.18306860327720642, + "1113268940/train2": 0.06608206033706665, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.974869817495346, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 0.4897426515817642, + "1231780997/train1": 0.0, + "1231780997/train2": 0.5006794333457947, + "1271718007/train0": 0.0, + "1271718007/train1": 0.00353144109249115, + "1271718007/train2": 0.19734752923250198, + "1273247072/train0": 0.0, + "1273247072/train1": 0.45726464316248894, + "1273247072/train2": 0.35906313359737396, + "1358421325/train0": 0.5775325745344162, + "1358421325/train1": 0.6467066705226898, + "1358421325/train2": 1.90096914768219, + "1375492786/train0": 0.0002854267368093133, "1375492786/train1": 0.00047226750757545233, "1375492786/train2": 0.0002880295505747199, "141984615/train0": 0.03316789741317431, "141984615/train1": 0.05730179324746132, "141984615/train2": 0.013830550014972687, "1475440973/train0": 0.0, - "1475440973/train1": 1.289921224117279, - "1475440973/train2": 1.28389310836792, + "1475440973/train1": 1.289921224117279, + "1475440973/train2": 1.28389310836792, + "1483645336/train0": 0.0, + "1483645336/train1": 1.9307887554168701, + "1483645336/train2": 1.9686058908700943, + "1522272686/train0": 0.009524375200271606, + "1522272686/train1": 0.03616611659526825, + "1522272686/train2": 0.017972871661186218, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 1.280472755432129, + "1549324805/train0": 0.0034310482442378998, + "1549324805/train1": 0.0015795694974561532, + "1549324805/train2": 0.01237945475926002, + "1607773860/train0": 1.779732346534729, + "1607773860/train1": 0.6036040186882019, + "1607773860/train2": 0.0, + "1665739511/train0": 0.007551252841949463, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.027157030999660492, + "1747178030/train2": 0.034092940390110016, + "1881223807/train0": 0.020236089825630188, + "1881223807/train1": 0.009028241038322449, + "1881223807/train2": 0.024854734539985657, + "1919198188/train0": 0.016854479908943176, + "1919198188/train1": 0.0, + "1919198188/train2": 0.018839523196220398, + "1991414747/train0": 0.0, + "1991414747/train1": 1.9863876849412918, + "1991414747/train2": 0.0, + "1997090716/train0": 1.9691698104143143, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.01770364921540022, + "2015956957/train1": 0.022195855714380743, + "2015956957/train2": 0.005573729798197746, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.5908347219228745, + "211245842/train1": 0.6792904287576675, + "211245842/train2": 0.0, + "2141802816/train0": 0.07543906569480896, + "2141802816/train1": 0.019766509532928467, + "2141802816/train2": 1.861636370420456, + "256770874/train0": 0.10820980370044708, + "256770874/train1": 0.7044661641120911, + "256770874/train2": 0.046216949820518494, + "278008524/train0": 0.0016456245793960989, + "278008524/train1": 0.011409573198761791, + "278008524/train2": 0.020201382460072637, + "416183106/train0": 0.3360496461391449, + "416183106/train1": 1.7668711841106415, + "416183106/train2": 1.812951534986496, + "534717141/train0": 1.9274959191679955, + "534717141/train1": 1.9558932781219482, + "534717141/train2": 0.0, + "551504635/train0": 0.033253490924835205, + "551504635/train1": 0.0, + "551504635/train2": 1.273692786693573, + "565369392/train0": 0.06443698455890019, + "565369392/train1": 0.046260833740234375, + "565369392/train2": 0.04557279869914055, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.043168901621053614, + "585904130/train1": 0.012068324722349644, + "585904130/train2": 0.011032561771571636, + "626486813/train0": 0.018570441869087517, + "626486813/train1": 0.02500777377281338, + "626486813/train2": 0.031722720130346715, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.006491700187325478, + "706479600/train0": 1.1865743398666382, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.07732226233929396, + "750448296/train1": 0.02701035887002945, + "750448296/train2": 0.01770863775163889, + "75070641/train0": 0.07482179999351501, + "75070641/train1": 0.5863077491521835, + "75070641/train2": 0.3710349500179291, + "767747424/train0": 0.015960504300892353, + "767747424/train1": 0.04561196081340313, + "767747424/train2": 0.024658456211909652, + "77147264/train0": 1.868594542145729, + "77147264/train1": 0.0, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 0.0, + "957101489/train1": 1.7757539749145508, + "957101489/train2": 1.432861328125, + "998761162/train0": 0.07749588787555695, + "998761162/train1": 1.9512543678283691, + "998761162/train2": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.5390796998678109, + "low": 0.18723032816778867, + "mean": 0.3798908010369334, + "n": 5, + "num_samples": 500 + }, + "mean": 0.3798908010369334, + "per_task": { + "LiftPegUpright-v1": 0.516516399052408, + "PickCube-v1": 0.2721092589199543, + "PullCube-v1": 0.05991193203954026, + "PushCube-v1": 0.4573507461792383, + "StackCube-v1": 0.593565668993526 + } + }, + "metric": "selector_regret_at_8", + "micro": { + "confidence": 0.95, + "high": 0.37381450664971905, + "low": 0.2099735496069669, + "mean": 0.293583450403069, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "success_selector_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.2624113475177305, + "low": 0.1276595744680851, + "mean": 0.19148936170212766, + "n": 141, + "num_samples": 500 + }, + "mean": 0.19148936170212766, + "per_seed": { + "1025740212/train0": 0.0, + "1025740212/train1": 0.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 0.0, + "1055157354/train2": 0.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 0.0, + "1113268940/train1": 0.0, + "1113268940/train2": 0.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 0.0, + "1231780997/train1": 0.0, + "1231780997/train2": 0.0, + "1271718007/train0": 0.0, + "1271718007/train1": 0.0, + "1271718007/train2": 0.0, + "1273247072/train0": 0.0, + "1273247072/train1": 0.0, + "1273247072/train2": 0.0, + "1358421325/train0": 0.0, + "1358421325/train1": 0.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, + "1483645336/train0": 0.0, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 0.0, + "1522272686/train1": 0.0, + "1522272686/train2": 0.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.0, + "1549324805/train2": 0.0, + "1607773860/train0": 1.0, + "1607773860/train1": 0.0, + "1607773860/train2": 0.0, + "1665739511/train0": 0.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 0.0, + "1747178030/train2": 0.0, + "1881223807/train0": 0.0, + "1881223807/train1": 0.0, + "1881223807/train2": 0.0, + "1919198188/train0": 0.0, + "1919198188/train1": 0.0, + "1919198188/train2": 0.0, + "1991414747/train0": 0.0, + "1991414747/train1": 1.0, + "1991414747/train2": 0.0, + "1997090716/train0": 1.0, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.0, + "2015956957/train1": 0.0, + "2015956957/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 0.0, + "211245842/train1": 0.0, + "211245842/train2": 0.0, + "2141802816/train0": 0.0, + "2141802816/train1": 0.0, + "2141802816/train2": 1.0, + "256770874/train0": 0.0, + "256770874/train1": 0.0, + "256770874/train2": 0.0, + "278008524/train0": 0.0, + "278008524/train1": 0.0, + "278008524/train2": 0.0, + "416183106/train0": 0.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, + "534717141/train2": 0.0, + "551504635/train0": 0.0, + "551504635/train1": 0.0, + "551504635/train2": 1.0, + "565369392/train0": 0.0, + "565369392/train1": 0.0, + "565369392/train2": 0.0, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.0, + "626486813/train2": 0.0, + "636680137/train0": 0.0, + "636680137/train1": 0.0, + "636680137/train2": 0.0, + "706479600/train0": 1.0, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 0.0, + "75070641/train1": 0.0, + "75070641/train2": 0.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 0.0, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 0.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 0.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.2594871794871795, + "low": 0.05725490196078431, + "mean": 0.15896430367018605, + "n": 5, + "num_samples": 500 + }, + "mean": 0.15896430367018605, + "per_task": { + "LiftPegUpright-v1": 0.2777777777777778, + "PickCube-v1": 0.0, + "PullCube-v1": 0.0196078431372549, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.26666666666666666 + } + }, + "metric": "success_selector_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.18840579710144928, + "low": 0.08695652173913043, + "mean": 0.13526570048309178, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "success_support_gap_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.43522458628841615, + "low": 0.2907801418439716, + "mean": 0.3619385342789597, + "n": 141, + "num_samples": 500 + }, + "mean": 0.3619385342789597, + "per_seed": { + "1025740212/train0": 1.0, + "1025740212/train1": 1.0, + "1025740212/train2": 0.0, + "1055157354/train0": 0.0, + "1055157354/train1": 1.0, + "1055157354/train2": 1.0, + "1055567618/train0": 0.0, + "1055567618/train1": 0.0, + "1055567618/train2": 0.0, + "1113268940/train0": 1.0, + "1113268940/train1": 1.0, + "1113268940/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 0.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 1.0, + "1231780997/train1": 0.0, + "1231780997/train2": 1.0, + "1271718007/train0": 0.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 1.0, + "1273247072/train1": 1.0, + "1273247072/train2": 1.0, + "1358421325/train0": 1.0, + "1358421325/train1": 1.0, + "1358421325/train2": 0.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 0.0, + "1475440973/train2": 0.0, + "1483645336/train0": 0.0, + "1483645336/train1": 0.0, + "1483645336/train2": 0.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 0.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 0.0, + "1607773860/train1": 1.0, + "1607773860/train2": 0.0, + "1665739511/train0": 1.0, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1881223807/train0": 1.0, + "1881223807/train1": 1.0, + "1881223807/train2": 1.0, + "1919198188/train0": 1.0, + "1919198188/train1": 1.0, + "1919198188/train2": 1.0, + "1991414747/train0": 0.0, + "1991414747/train1": 0.0, + "1991414747/train2": 0.0, + "1997090716/train0": 0.0, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.0, + "2015956957/train1": 0.2, + "2015956957/train2": 0.0, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 1.0, + "211245842/train1": 1.0, + "211245842/train2": 0.0, + "2141802816/train0": 1.0, + "2141802816/train1": 1.0, + "2141802816/train2": 0.0, + "256770874/train0": 1.0, + "256770874/train1": 1.0, + "256770874/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "416183106/train0": 1.0, + "416183106/train1": 0.0, + "416183106/train2": 0.0, + "534717141/train0": 0.0, + "534717141/train1": 0.0, + "534717141/train2": 0.0, + "551504635/train0": 1.0, + "551504635/train1": 1.0, + "551504635/train2": 0.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.6666666666666666, + "565369392/train2": 0.6666666666666666, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 0.0, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 1.0, + "75070641/train1": 1.0, + "75070641/train2": 1.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 0.0, + "77147264/train1": 0.5, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 1.0, + "957101489/train1": 0.0, + "957101489/train2": 0.0, + "998761162/train0": 1.0, + "998761162/train1": 0.0, + "998761162/train2": 0.0 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 0.7572649572649572, + "low": 0.24550025138260434, + "mean": 0.48667672197083967, + "n": 5, + "num_samples": 500 + }, + "mean": 0.48667672197083967, + "per_task": { + "LiftPegUpright-v1": 0.5, + "PickCube-v1": 1.0, + "PullCube-v1": 0.14705882352941177, + "PushCube-v1": 0.23076923076923078, + "StackCube-v1": 0.5555555555555556 + } + }, + "metric": "success_support_gap_at_8", + "micro": { + "confidence": 0.95, + "high": 0.36231884057971014, + "low": 0.2318840579710145, + "mean": 0.2946859903381642, + "n": 207, + "num_samples": 500 + }, + "n": 207 + }, + "success_total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 0.6320330969267138, + "low": 0.47304964539007094, + "mean": 0.5534278959810874, + "n": 141, + "num_samples": 500 + }, + "mean": 0.5534278959810874, + "per_seed": { + "1025740212/train0": 1.0, + "1025740212/train1": 1.0, + "1025740212/train2": 1.0, + "1055157354/train0": 1.0, + "1055157354/train1": 1.0, + "1055157354/train2": 1.0, + "1055567618/train0": 1.0, + "1055567618/train1": 1.0, + "1055567618/train2": 1.0, + "1113268940/train0": 1.0, + "1113268940/train1": 1.0, + "1113268940/train2": 1.0, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.0, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 1.0, + "1231780997/train1": 0.0, + "1231780997/train2": 1.0, + "1271718007/train0": 0.0, + "1271718007/train1": 0.5, + "1271718007/train2": 0.5, + "1273247072/train0": 1.0, + "1273247072/train1": 1.0, + "1273247072/train2": 1.0, + "1358421325/train0": 1.0, + "1358421325/train1": 1.0, + "1358421325/train2": 1.0, + "1375492786/train0": 0.0, + "1375492786/train1": 0.0, + "1375492786/train2": 0.0, + "141984615/train0": 0.0, + "141984615/train1": 0.0, + "141984615/train2": 0.0, + "1475440973/train0": 0.0, + "1475440973/train1": 1.0, + "1475440973/train2": 1.0, "1483645336/train0": 0.0, - "1483645336/train1": 1.9307887554168701, - "1483645336/train2": 1.9686058908700943, - "1522272686/train0": 0.009524375200271606, - "1522272686/train1": 0.03616611659526825, - "1522272686/train2": 0.017972871661186218, + "1483645336/train1": 1.0, + "1483645336/train2": 1.0, + "1522272686/train0": 1.0, + "1522272686/train1": 1.0, + "1522272686/train2": 1.0, "1528740417/train0": 0.0, "1528740417/train1": 0.0, - "1528740417/train2": 1.280472755432129, - "1549324805/train0": 0.0034310482442378998, - "1549324805/train1": 0.0015795694974561532, - "1549324805/train2": 0.01237945475926002, - "1607773860/train0": 1.779732346534729, - "1607773860/train1": 0.6036040186882019, + "1528740417/train2": 1.0, + "1549324805/train0": 0.0, + "1549324805/train1": 0.3333333333333333, + "1549324805/train2": 0.3333333333333333, + "1607773860/train0": 1.0, + "1607773860/train1": 1.0, "1607773860/train2": 0.0, - "1665739511/train0": 0.007551252841949463, + "1665739511/train0": 1.0, "1665739511/train1": 0.0, "1665739511/train2": 0.0, "1747178030/train0": 0.0, - "1747178030/train1": 0.027157030999660492, - "1747178030/train2": 0.034092940390110016, - "1881223807/train0": 0.020236089825630188, - "1881223807/train1": 0.009028241038322449, - "1881223807/train2": 0.024854734539985657, - "1919198188/train0": 0.016854479908943176, - "1919198188/train1": 0.0, - "1919198188/train2": 0.018839523196220398, + "1747178030/train1": 1.0, + "1747178030/train2": 1.0, + "1881223807/train0": 1.0, + "1881223807/train1": 1.0, + "1881223807/train2": 1.0, + "1919198188/train0": 1.0, + "1919198188/train1": 1.0, + "1919198188/train2": 1.0, "1991414747/train0": 0.0, - "1991414747/train1": 1.9863876849412918, + "1991414747/train1": 1.0, "1991414747/train2": 0.0, - "1997090716/train0": 1.9691698104143143, + "1997090716/train0": 1.0, "1997090716/train1": 0.0, "1997090716/train2": 0.0, - "2015956957/train0": 0.01770364921540022, - "2015956957/train1": 0.022195855714380743, - "2015956957/train2": 0.005573729798197746, + "2015956957/train0": 0.0, + "2015956957/train1": 0.2, + "2015956957/train2": 0.0, "2054899126/train0": 0.0, "2054899126/train1": 0.0, "2054899126/train2": 0.0, - "211245842/train0": 0.5908347219228745, - "211245842/train1": 0.6792904287576675, + "211245842/train0": 1.0, + "211245842/train1": 1.0, "211245842/train2": 0.0, - "2141802816/train0": 0.07543906569480896, - "2141802816/train1": 0.019766509532928467, - "2141802816/train2": 1.861636370420456, - "256770874/train0": 0.10820980370044708, - "256770874/train1": 0.7044661641120911, - "256770874/train2": 0.046216949820518494, - "278008524/train0": 0.0016456245793960989, - "278008524/train1": 0.011409573198761791, - "278008524/train2": 0.020201382460072637, - "416183106/train0": 0.3360496461391449, - "416183106/train1": 1.7668711841106415, - "416183106/train2": 1.812951534986496, - "534717141/train0": 1.9274959191679955, - "534717141/train1": 1.9558932781219482, + "2141802816/train0": 1.0, + "2141802816/train1": 1.0, + "2141802816/train2": 1.0, + "256770874/train0": 1.0, + "256770874/train1": 1.0, + "256770874/train2": 1.0, + "278008524/train0": 0.5, + "278008524/train1": 0.5, + "278008524/train2": 0.5, + "416183106/train0": 1.0, + "416183106/train1": 1.0, + "416183106/train2": 1.0, + "534717141/train0": 1.0, + "534717141/train1": 1.0, "534717141/train2": 0.0, - "551504635/train0": 0.033253490924835205, - "551504635/train1": 0.0, - "551504635/train2": 1.273692786693573, - "565369392/train0": 0.06443698455890019, - "565369392/train1": 0.046260833740234375, - "565369392/train2": 0.04557279869914055, + "551504635/train0": 1.0, + "551504635/train1": 1.0, + "551504635/train2": 1.0, + "565369392/train0": 0.3333333333333333, + "565369392/train1": 0.6666666666666666, + "565369392/train2": 0.6666666666666666, "565621826/train0": 0.0, "565621826/train1": 0.0, "565621826/train2": 0.0, - "585904130/train0": 0.043168901621053614, - "585904130/train1": 0.012068324722349644, - "585904130/train2": 0.011032561771571636, - "626486813/train0": 0.018570441869087517, - "626486813/train1": 0.02500777377281338, - "626486813/train2": 0.031722720130346715, - "636680137/train0": 0.0, - "636680137/train1": 0.0, - "636680137/train2": 0.006491700187325478, - "706479600/train0": 1.1865743398666382, + "585904130/train0": 0.0, + "585904130/train1": 0.0, + "585904130/train2": 0.0, + "626486813/train0": 0.0, + "626486813/train1": 0.25, + "626486813/train2": 0.25, + "636680137/train0": 1.0, + "636680137/train1": 1.0, + "636680137/train2": 1.0, + "706479600/train0": 1.0, "706479600/train1": 0.0, "706479600/train2": 0.0, - "750448296/train0": 0.07732226233929396, - "750448296/train1": 0.02701035887002945, - "750448296/train2": 0.01770863775163889, - "75070641/train0": 0.07482179999351501, - "75070641/train1": 0.5863077491521835, - "75070641/train2": 0.3710349500179291, - "767747424/train0": 0.015960504300892353, - "767747424/train1": 0.04561196081340313, - "767747424/train2": 0.024658456211909652, - "77147264/train0": 1.868594542145729, - "77147264/train1": 0.0, + "750448296/train0": 0.0, + "750448296/train1": 0.0, + "750448296/train2": 0.0, + "75070641/train0": 1.0, + "75070641/train1": 1.0, + "75070641/train2": 1.0, + "767747424/train0": 0.0, + "767747424/train1": 0.0, + "767747424/train2": 0.0, + "77147264/train0": 1.0, + "77147264/train1": 0.5, "77147264/train2": 0.0, "794181626/train0": 0.0, "794181626/train1": 0.0, "794181626/train2": 0.0, - "957101489/train0": 0.0, - "957101489/train1": 1.7757539749145508, - "957101489/train2": 1.432861328125, - "998761162/train0": 0.07749588787555695, - "998761162/train1": 1.9512543678283691, - "998761162/train2": 1.939187914133072 + "957101489/train0": 1.0, + "957101489/train1": 1.0, + "957101489/train2": 1.0, + "998761162/train0": 1.0, + "998761162/train1": 1.0, + "998761162/train2": 1.0 } }, "macro_by_task": { "ci": { "confidence": 0.95, - "high": 0.5319262530406317, - "low": 0.19367229081819662, - "mean": 0.3798908010369334, + "high": 0.8844444444444444, + "low": 0.3478632478632479, + "mean": 0.6456410256410257, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, - "mean": 0.3798908010369334, + "mean": 0.6456410256410257, "per_task": { - "LiftPegUpright-v1": 0.516516399052408, - "PickCube-v1": 0.2721092589199543, - "PullCube-v1": 0.05991193203954026, - "PushCube-v1": 0.4573507461792383, - "StackCube-v1": 0.593565668993526 + "LiftPegUpright-v1": 0.7777777777777778, + "PickCube-v1": 1.0, + "PullCube-v1": 0.16666666666666666, + "PushCube-v1": 0.46153846153846156, + "StackCube-v1": 0.8222222222222222 } }, - "metric": "selector_regret_at_8", + "metric": "success_total_car_to_hidden_at_8", "micro": { "confidence": 0.95, - "high": 0.3721372773162211, - "low": 0.21313571063115527, - "mean": 0.293583450403069, + "high": 0.4975845410628019, + "low": 0.357487922705314, + "mean": 0.42995169082125606, "n": 207, - "num_samples": 1000 + "num_samples": 500 }, "n": 207 }, @@ -749,11 +3524,11 @@ "macro_by_seed": { "ci": { "confidence": 0.95, - "high": 0.7276497844564805, - "low": 0.49026043312640605, + "high": 0.7238702659522698, + "low": 0.4987892105245752, "mean": 0.6062044484887313, "n": 141, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.6062044484887313, "per_seed": { @@ -907,7 +3682,7 @@ "low": 0.4585766474838634, "mean": 0.788961660718036, "n": 5, - "num_samples": 1000 + "num_samples": 500 }, "mean": 0.788961660718036, "per_task": { @@ -921,11 +3696,195 @@ "metric": "support_gap_at_8", "micro": { "confidence": 0.95, - "high": 0.6340587634713039, + "high": 0.6411033530127941, "low": 0.4234259423213799, "mean": 0.5282977284379371, "n": 207, - "num_samples": 1000 + "num_samples": 500 + }, + "n": 207 + }, + "total_car_to_hidden_at_8": { + "macro_by_seed": { + "ci": { + "confidence": 0.95, + "high": 1.1419460938234778, + "low": 0.8777518347707955, + "mean": 1.0115917835033712, + "n": 141, + "num_samples": 500 + }, + "mean": 1.0115917835033712, + "per_seed": { + "1025740212/train0": 1.7857587486505508, + "1025740212/train1": 1.7790512591600418, + "1025740212/train2": 1.772506669163704, + "1055157354/train0": 1.8516457676887512, + "1055157354/train1": 1.8137879967689514, + "1055157354/train2": 1.8101318180561066, + "1055567618/train0": 1.183156132698059, + "1055567618/train1": 1.8253512382507324, + "1055567618/train2": 1.9218642711639404, + "1113268940/train0": 1.7200938165187836, + "1113268940/train1": 1.5213188230991364, + "1113268940/train2": 1.3635571599006653, + "1113371261/train0": 0.0, + "1113371261/train1": 0.0, + "1113371261/train2": 0.0, + "1176943004/train0": 0.0, + "1176943004/train1": 1.974869817495346, + "1176943004/train2": 0.0, + "1189392917/train0": 0.0, + "1189392917/train1": 0.0, + "1189392917/train2": 0.0, + "1231780997/train0": 1.7710015326738358, + "1231780997/train1": 0.0, + "1231780997/train2": 1.7728382349014282, + "1271718007/train0": 0.0, + "1271718007/train1": 0.89248376339674, + "1271718007/train2": 0.9337802603840828, + "1273247072/train0": 1.8672154396772385, + "1273247072/train1": 1.9575153104960918, + "1273247072/train2": 1.8952929228544235, + "1358421325/train0": 1.7632866352796555, + "1358421325/train1": 1.8191632330417633, + "1358421325/train2": 1.90096914768219, + "1375492786/train0": 0.2420923411846161, + "1375492786/train1": 0.24213319027330726, + "1375492786/train2": 0.24285174411488697, + "141984615/train0": 0.15386936937769255, + "141984615/train1": 0.1653481237590313, + "141984615/train2": 0.1600442255536715, + "1475440973/train0": 0.0, + "1475440973/train1": 1.289921224117279, + "1475440973/train2": 1.28389310836792, + "1483645336/train0": 0.0, + "1483645336/train1": 1.9307887554168701, + "1483645336/train2": 1.9686058908700943, + "1522272686/train0": 1.7659689038991928, + "1522272686/train1": 1.8036767393350601, + "1522272686/train2": 1.8230728209018707, + "1528740417/train0": 0.0, + "1528740417/train1": 0.0, + "1528740417/train2": 1.280472755432129, + "1549324805/train0": 0.06341505361100037, + "1549324805/train1": 0.7258233395405114, + "1549324805/train2": 0.7317837377389272, + "1607773860/train0": 1.779732346534729, + "1607773860/train1": 1.7768966555595398, + "1607773860/train2": 0.0, + "1665739511/train0": 1.1859291791915894, + "1665739511/train1": 0.0, + "1665739511/train2": 0.0, + "1747178030/train0": 0.0, + "1747178030/train1": 1.915814757347107, + "1747178030/train2": 1.9092875719070435, + "1881223807/train0": 1.9688839614391327, + "1881223807/train1": 1.9741420596837997, + "1881223807/train2": 1.9813720285892487, + "1919198188/train0": 1.9269265234470367, + "1919198188/train1": 1.9537556618452072, + "1919198188/train2": 1.9560884833335876, + "1991414747/train0": 0.0, + "1991414747/train1": 1.9863876849412918, + "1991414747/train2": 0.0, + "1997090716/train0": 1.9691698104143143, + "1997090716/train1": 0.0, + "1997090716/train2": 0.0, + "2015956957/train0": 0.09634189326316119, + "2015956957/train1": 0.49578512627631427, + "2015956957/train2": 0.09564426057040691, + "2054899126/train0": 0.0, + "2054899126/train1": 0.0, + "2054899126/train2": 0.0, + "211245842/train0": 1.7778205126523972, + "211245842/train1": 1.843446210026741, + "211245842/train2": 0.0, + "2141802816/train0": 1.950709506869316, + "2141802816/train1": 1.9238732755184174, + "2141802816/train2": 1.861636370420456, + "256770874/train0": 1.86713507771492, + "256770874/train1": 1.891966164112091, + "256770874/train2": 1.835601955652237, + "278008524/train0": 1.1040238340501674, + "278008524/train1": 1.0937489594216458, + "278008524/train2": 1.1011469289660454, + "416183106/train0": 1.7582354247570038, + "416183106/train1": 1.7668711841106415, + "416183106/train2": 1.812951534986496, + "534717141/train0": 1.9274959191679955, + "534717141/train1": 1.9558932781219482, + "534717141/train2": 0.0, + "551504635/train0": 1.3081637024879456, + "551504635/train1": 1.2775209546089172, + "551504635/train2": 1.273692786693573, + "565369392/train0": 0.7057971333463987, + "565369392/train1": 1.2992276052633922, + "565369392/train2": 1.339533941199382, + "565621826/train0": 0.0, + "565621826/train1": 0.0, + "565621826/train2": 0.0, + "585904130/train0": 0.2248280394511918, + "585904130/train1": 0.2167967443043987, + "585904130/train2": 0.1701989375675718, + "626486813/train0": 0.08356922247912735, + "626486813/train1": 0.5638198243686929, + "626486813/train2": 0.5741560639580712, + "636680137/train0": 1.9875853462144732, + "636680137/train1": 1.986071527004242, + "636680137/train2": 1.9821314811706543, + "706479600/train0": 1.1865743398666382, + "706479600/train1": 0.0, + "706479600/train2": 0.0, + "750448296/train0": 0.20613667648285627, + "750448296/train1": 0.18939130753278732, + "750448296/train2": 0.1865733889862895, + "75070641/train0": 1.8351471424102783, + "75070641/train1": 1.7734202593564987, + "75070641/train2": 1.805951327085495, + "767747424/train0": 0.0750129884108901, + "767747424/train1": 0.12081881728954613, + "767747424/train2": 0.10090384539216757, + "77147264/train0": 1.868594542145729, + "77147264/train1": 0.573363870382309, + "77147264/train2": 0.0, + "794181626/train0": 0.0, + "794181626/train1": 0.0, + "794181626/train2": 0.0, + "957101489/train0": 1.4118301272392273, + "957101489/train1": 1.7757539749145508, + "957101489/train2": 1.432861328125, + "998761162/train0": 1.993456482887268, + "998761162/train1": 1.9512543678283691, + "998761162/train2": 1.939187914133072 + } + }, + "macro_by_task": { + "ci": { + "confidence": 0.95, + "high": 1.6102036754837912, + "low": 0.722751199278172, + "mean": 1.1686045027220224, + "n": 5, + "num_samples": 500 + }, + "mean": 1.1686045027220224, + "per_task": { + "LiftPegUpright-v1": 1.1777661542097728, + "PickCube-v1": 1.9066745576759179, + "PullCube-v1": 0.42758709733321915, + "PushCube-v1": 0.9042525268518008, + "StackCube-v1": 1.4267421775394016 + } + }, + "metric": "total_car_to_hidden_at_8", + "micro": { + "confidence": 0.95, + "high": 0.9329092862770065, + "low": 0.6954710805647476, + "mean": 0.8212702652815715, + "n": 207, + "num_samples": 500 }, "n": 207 } diff --git a/workspace/runs/ctt_val_rollout_comparison/metrics_by_seed.json b/workspace/runs/ctt_val_rollout_comparison/metrics_by_seed.json new file mode 100644 index 0000000000000000000000000000000000000000..3d0fc6905b8a769e3be28046bd3e54b589fe86d3 --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/metrics_by_seed.json @@ -0,0 +1,3245 @@ +{ + "1025740212/train0": { + "base_success": 0.0, + "base_utility": 0.528373658657074, + "branch_car_at_8": 0.26188503205776215, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.08963387331564551, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4761262834072113, + "proposal_oracle_utility_gain_over_base_at_8": -0.05224737524986267, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.21424125134944916, + "selected_utility_gain_over_base_at_8": -0.3141324073076248, + "selector_regret_at_8": 0.26188503205776215, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5238737165927887, + "total_car_to_hidden_at_8": 1.7857587486505508 + }, + "1025740212/train1": { + "base_success": 0.0, + "base_utility": 0.528373658657074, + "branch_car_at_8": 0.47739143669605255, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.12992986623548633, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6983401775360107, + "proposal_oracle_utility_gain_over_base_at_8": 0.16996651887893677, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2209487408399582, + "selected_utility_gain_over_base_at_8": -0.3074249178171158, + "selector_regret_at_8": 0.47739143669605255, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3016598224639893, + "total_car_to_hidden_at_8": 1.7790512591600418 + }, + "1025740212/train2": { + "base_success": 0.0, + "base_utility": 0.528373658657074, + "branch_car_at_8": 1.772506669163704, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.17388092747958847, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.471626341342926, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22749333083629608, + "selected_utility_gain_over_base_at_8": -0.3008803278207779, + "selector_regret_at_8": 1.772506669163704, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.772506669163704 + }, + "1055157354/train0": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, + "branch_car_at_8": 1.8516457676887512, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2973966626262834, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8235089033842087, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.14835423231124878, + "selected_utility_gain_over_base_at_8": -0.02813686430454254, + "selector_regret_at_8": 1.8516457676887512, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8516457676887512 + }, + "1055157354/train1": { + "base_success": 0.0, + "base_utility": 0.17649109661579132, + "branch_car_at_8": 0.6370042562484741, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07450750352404123, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8232162594795227, + "proposal_oracle_utility_gain_over_base_at_8": 0.6467251628637314, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18621200323104858, + "selected_utility_gain_over_base_at_8": 0.009720906615257263, + "selector_regret_at_8": 0.6370042562484741, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1767837405204773, + "total_car_to_hidden_at_8": 1.8137879967689514 + }, + "1055157354/train2": { + "base_success": 0.0, + "base_utility": 0.19648407399654388, + "branch_car_at_8": 0.03480546176433563, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1805709977888097, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.22467364370822906, + "proposal_oracle_utility_gain_over_base_at_8": 0.02818956971168518, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18986818194389343, + "selected_utility_gain_over_base_at_8": -0.006615892052650452, + "selector_regret_at_8": 0.03480546176433563, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.775326356291771, + "total_car_to_hidden_at_8": 1.8101318180561066 + }, + "1055567618/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.183156132698059, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07486610854656817, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8168438673019409, + "selected_utility_gain_over_base_at_8": -1.183156132698059, + "selector_regret_at_8": 1.183156132698059, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.183156132698059 + }, + "1055567618/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.8253512382507324, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.318234334809661, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.17464876174926758, + "selected_utility_gain_over_base_at_8": -1.8253512382507324, + "selector_regret_at_8": 1.8253512382507324, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.8253512382507324 + }, + "1055567618/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9218642711639404, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19289744165908876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07813572883605957, + "selected_utility_gain_over_base_at_8": -1.9218642711639404, + "selector_regret_at_8": 1.9218642711639404, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9218642711639404 + }, + "1113268940/train0": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, + "branch_car_at_8": 0.38470593094825745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1616344352768096, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6646121144294739, + "proposal_oracle_utility_gain_over_base_at_8": 0.24229958653450012, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.27990618348121643, + "selected_utility_gain_over_base_at_8": -0.14240634441375732, + "selector_regret_at_8": 0.38470593094825745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.3353878855705261, + "total_car_to_hidden_at_8": 1.7200938165187836 + }, + "1113268940/train1": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, + "branch_car_at_8": 0.18306860327720642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10725283874677279, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.6617497801780701, + "proposal_oracle_utility_gain_over_base_at_8": 0.2394372522830963, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.47868117690086365, + "selected_utility_gain_over_base_at_8": 0.05636864900588989, + "selector_regret_at_8": 0.18306860327720642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.33825021982193, + "total_car_to_hidden_at_8": 1.5213188230991364 + }, + "1113268940/train2": { + "base_success": 0.0, + "base_utility": 0.42231252789497375, + "branch_car_at_8": 0.06608206033706665, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.032532089835823584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7025249004364014, + "proposal_oracle_utility_gain_over_base_at_8": 0.2802123725414276, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6364428400993347, + "selected_utility_gain_over_base_at_8": 0.21413031220436096, + "selector_regret_at_8": 0.06608206033706665, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2974750995635986, + "total_car_to_hidden_at_8": 1.3635571599006653 + }, + "1113371261/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.3179396489681421, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1113371261/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.04977883974023356, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1113371261/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.233029795067053, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1176943004/train0": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2176350935010736, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1176943004/train1": { + "base_success": 0.0, + "base_utility": 0.058033183217048645, + "branch_car_at_8": 1.974869817495346, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1815411965846294, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9419668167829514, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02513018250465393, + "selected_utility_gain_over_base_at_8": -0.032903000712394714, + "selector_regret_at_8": 1.974869817495346, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.974869817495346 + }, + "1176943004/train2": { + "base_success": 0.0, + "base_utility": 0.05094052851200104, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.17468938592004596, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.949059471487999, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.949059471487999, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1189392917/train0": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1628408162009477, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1189392917/train1": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2180235973132697, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1189392917/train2": { + "base_success": 0.0, + "base_utility": 0.10841864347457886, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8915813565254211, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8915813565254211, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1231780997/train0": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, + "branch_car_at_8": 0.4897426515817642, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.18322944379343714, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7187411189079285, + "proposal_oracle_utility_gain_over_base_at_8": -0.008265018463134766, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22899846732616425, + "selected_utility_gain_over_base_at_8": -0.498007670044899, + "selector_regret_at_8": 0.4897426515817642, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2812588810920715, + "total_car_to_hidden_at_8": 1.7710015326738358 + }, + "1231780997/train1": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28322114525972475, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.2729938626289368, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.2729938626289368, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1231780997/train2": { + "base_success": 0.0, + "base_utility": 0.7270061373710632, + "branch_car_at_8": 0.5006794333457947, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10207358838016511, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7278411984443665, + "proposal_oracle_utility_gain_over_base_at_8": 0.0008350610733032227, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22716176509857178, + "selected_utility_gain_over_base_at_8": -0.49984437227249146, + "selector_regret_at_8": 0.5006794333457947, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2721588015556335, + "total_car_to_hidden_at_8": 1.7728382349014282 + }, + "1271718007/train0": { + "base_success": 0.5, + "base_utility": 1.4142604768276215, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.07543261691159965, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5857395231723785, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5857395231723785, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1271718007/train1": { + "base_success": 0.5, + "base_utility": 1.415381133556366, + "branch_car_at_8": 0.00353144109249115, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07023931523066618, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.1110476776957512, + "proposal_oracle_utility_gain_over_base_at_8": -0.3043334558606148, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.10751623660326, + "selected_utility_gain_over_base_at_8": -0.3078648969531059, + "selector_regret_at_8": 0.00353144109249115, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.8889523223042488, + "total_car_to_hidden_at_8": 0.89248376339674 + }, + "1271718007/train2": { + "base_success": 0.5, + "base_utility": 1.415381133556366, + "branch_car_at_8": 0.19734752923250198, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2350719661460603, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.2635672688484192, + "proposal_oracle_utility_gain_over_base_at_8": -0.15181386470794678, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.0662197396159172, + "selected_utility_gain_over_base_at_8": -0.34916139394044876, + "selector_regret_at_8": 0.19734752923250198, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.7364327311515808, + "total_car_to_hidden_at_8": 0.9337802603840828 + }, + "1273247072/train0": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.15040783499409596, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.13278456032276154, + "proposal_oracle_utility_gain_over_base_at_8": -0.02064509689807892, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13278456032276154, + "selected_utility_gain_over_base_at_8": -0.02064509689807892, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8672154396772385, + "total_car_to_hidden_at_8": 1.8672154396772385 + }, + "1273247072/train1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.45726464316248894, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.096994407338536, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.4997493326663971, + "proposal_oracle_utility_gain_over_base_at_8": 0.34631967544555664, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04248468950390816, + "selected_utility_gain_over_base_at_8": -0.1109449677169323, + "selector_regret_at_8": 0.45726464316248894, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.500250667333603, + "total_car_to_hidden_at_8": 1.9575153104960918 + }, + "1273247072/train2": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.35906313359737396, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2793744975404514, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.46377021074295044, + "proposal_oracle_utility_gain_over_base_at_8": 0.31034055352211, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10470707714557648, + "selected_utility_gain_over_base_at_8": -0.04872258007526398, + "selector_regret_at_8": 0.35906313359737396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.5362297892570496, + "total_car_to_hidden_at_8": 1.8952929228544235 + }, + "1358421325/train0": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, + "branch_car_at_8": 0.5775325745344162, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.13502516765102715, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8142459392547607, + "proposal_oracle_utility_gain_over_base_at_8": -0.018456339836120605, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23671336472034454, + "selected_utility_gain_over_base_at_8": -0.5959889143705368, + "selector_regret_at_8": 0.5775325745344162, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1857540607452393, + "total_car_to_hidden_at_8": 1.7632866352796555 + }, + "1358421325/train1": { + "base_success": 0.0, + "base_utility": 0.8324764966964722, + "branch_car_at_8": 0.6467066705226898, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.046694794435454434, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8275434374809265, + "proposal_oracle_utility_gain_over_base_at_8": -0.004933059215545654, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.1808367669582367, + "selected_utility_gain_over_base_at_8": -0.6516397297382355, + "selector_regret_at_8": 0.6467066705226898, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1724565625190735, + "total_car_to_hidden_at_8": 1.8191632330417633 + }, + "1358421325/train2": { + "base_success": 0.0, + "base_utility": 0.8327022790908813, + "branch_car_at_8": 1.90096914768219, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.32948159974174274, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1672977209091187, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09903085231781006, + "selected_utility_gain_over_base_at_8": -0.7336714267730713, + "selector_regret_at_8": 1.90096914768219, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.90096914768219 + }, + "1375492786/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0002854267368093133, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19928218350064877, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0016440550098195672, + "proposal_oracle_utility_gain_over_base_at_8": -1.9983559449901804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001358628273010254, + "selected_utility_gain_over_base_at_8": -1.9986413717269897, + "selector_regret_at_8": 0.0002854267368093133, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24180691444780678, + "total_car_to_hidden_at_8": 0.2420923411846161 + }, + "1375492786/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.00047226750757545233, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.14027669602920534, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0017900466918945312, + "proposal_oracle_utility_gain_over_base_at_8": -1.9982099533081055, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.001317779184319079, + "selected_utility_gain_over_base_at_8": -1.998682220815681, + "selector_regret_at_8": 0.00047226750757545233, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.2416609227657318, + "total_car_to_hidden_at_8": 0.24213319027330726 + }, + "1375492786/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0002880295505747199, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.24345096945762634, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.008822272371479922, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.0008872548933140934, + "proposal_oracle_utility_gain_over_base_at_8": -1.999112745106686, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0005992253427393734, + "selected_utility_gain_over_base_at_8": -1.9994007746572606, + "selector_regret_at_8": 0.0002880295505747199, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.24256371456431225, + "total_car_to_hidden_at_8": 0.24285174411488697 + }, + "141984615/train0": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, + "branch_car_at_8": 0.03316789741317431, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.10926310602682258, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0797463705142339, + "proposal_oracle_utility_gain_over_base_at_8": -0.01406576856970787, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04657847310105959, + "selected_utility_gain_over_base_at_8": -0.04723366598288218, + "selector_regret_at_8": 0.03316789741317431, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12070147196451823, + "total_car_to_hidden_at_8": 0.15386936937769255 + }, + "141984615/train1": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, + "branch_car_at_8": 0.05730179324746132, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.08533231968443573, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09240151196718216, + "proposal_oracle_utility_gain_over_base_at_8": -0.001410627116759618, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03509971871972084, + "selected_utility_gain_over_base_at_8": -0.05871242036422094, + "selector_regret_at_8": 0.05730179324746132, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.11134201536575954, + "total_car_to_hidden_at_8": 0.1653481237590313 + }, + "141984615/train2": { + "base_success": 0.0, + "base_utility": 0.09381213908394177, + "branch_car_at_8": 0.013830550014972687, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.20044784247875214, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.19434281311769708, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.054234166940053306, + "proposal_oracle_utility_gain_over_base_at_8": -0.039577972143888474, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04040361692508062, + "selected_utility_gain_over_base_at_8": -0.05340852215886116, + "selector_regret_at_8": 0.013830550014972687, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.14621367553869882, + "total_car_to_hidden_at_8": 0.1600442255536715 + }, + "1475440973/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.09930428958964055, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1475440973/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.289921224117279, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2443552809335613, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.710078775882721, + "selected_utility_gain_over_base_at_8": -1.289921224117279, + "selector_regret_at_8": 1.289921224117279, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.289921224117279 + }, + "1475440973/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.28389310836792, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.22214884767353088, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7161068916320801, + "selected_utility_gain_over_base_at_8": -1.28389310836792, + "selector_regret_at_8": 1.28389310836792, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.28389310836792 + }, + "1483645336/train0": { + "base_success": 0.0, + "base_utility": 0.01641605794429779, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9835839420557022, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.9835839420557022, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1483645336/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9307887554168701, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.13752192684190806, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.06921124458312988, + "selected_utility_gain_over_base_at_8": -1.9307887554168701, + "selector_regret_at_8": 1.9307887554168701, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9307887554168701 + }, + "1483645336/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9686058908700943, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.5995435922806452, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.0313941091299057, + "selected_utility_gain_over_base_at_8": -1.9686058908700943, + "selector_regret_at_8": 1.9686058908700943, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9686058908700943 + }, + "1522272686/train0": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, + "branch_car_at_8": 0.009524375200271606, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.08257819446483214, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2435554713010788, + "proposal_oracle_utility_gain_over_base_at_8": 0.048794373869895935, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.2340310961008072, + "selected_utility_gain_over_base_at_8": 0.03926999866962433, + "selector_regret_at_8": 0.009524375200271606, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7564445286989212, + "total_car_to_hidden_at_8": 1.7659689038991928 + }, + "1522272686/train1": { + "base_success": 0.0, + "base_utility": 0.19844311475753784, + "branch_car_at_8": 0.03616611659526825, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28555947838938045, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.23248937726020813, + "proposal_oracle_utility_gain_over_base_at_8": 0.03404626250267029, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.19632326066493988, + "selected_utility_gain_over_base_at_8": -0.0021198540925979614, + "selector_regret_at_8": 0.03616611659526825, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7675106227397919, + "total_car_to_hidden_at_8": 1.8036767393350601 + }, + "1522272686/train2": { + "base_success": 0.0, + "base_utility": 0.19476109743118286, + "branch_car_at_8": 0.017972871661186218, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.3276128524355476, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1949000507593155, + "proposal_oracle_utility_gain_over_base_at_8": 0.0001389533281326294, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.17692717909812927, + "selected_utility_gain_over_base_at_8": -0.01783391833305359, + "selector_regret_at_8": 0.017972871661186218, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8050999492406845, + "total_car_to_hidden_at_8": 1.8230728209018707 + }, + "1528740417/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.015990866806051107, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1528740417/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2757671830981365, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1528740417/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.280472755432129, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.16399362968800954, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.7195272445678711, + "selected_utility_gain_over_base_at_8": -1.280472755432129, + "selector_regret_at_8": 1.280472755432129, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.280472755432129 + }, + "1549324805/train0": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, + "branch_car_at_8": 0.0034310482442378998, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.19382954770507574, + "proposal_oracle_success_at_8": 0.6666666666666666, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.3427665165315072, + "proposal_oracle_utility_gain_over_base_at_8": -0.035980931793649994, + "selected_success_at_8": 0.6666666666666666, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.3393354682872693, + "selected_utility_gain_over_base_at_8": -0.03941198003788789, + "selector_regret_at_8": 0.0034310482442378998, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05998400536676248, + "total_car_to_hidden_at_8": 0.06341505361100037 + }, + "1549324805/train1": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, + "branch_car_at_8": 0.0015795694974561532, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.28625156462411294, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6785067518552145, + "proposal_oracle_utility_gain_over_base_at_8": -0.7002406964699427, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6769271823577583, + "selected_utility_gain_over_base_at_8": -0.7018202659673989, + "selector_regret_at_8": 0.0015795694974561532, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7242437700430552, + "total_car_to_hidden_at_8": 0.7258233395405114 + }, + "1549324805/train2": { + "base_success": 0.6666666666666666, + "base_utility": 1.3787474483251572, + "branch_car_at_8": 0.01237945475926002, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4027505218982697, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.23864987006465407, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.6833462389186025, + "proposal_oracle_utility_gain_over_base_at_8": -0.6954012094065547, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.6709667841593424, + "selected_utility_gain_over_base_at_8": -0.7077806641658148, + "selector_regret_at_8": 0.01237945475926002, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.7194042829796672, + "total_car_to_hidden_at_8": 0.7317837377389272 + }, + "1607773860/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.779732346534729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.08588083120555855, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.220267653465271, + "selected_utility_gain_over_base_at_8": -1.779732346534729, + "selector_regret_at_8": 1.779732346534729, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.779732346534729 + }, + "1607773860/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.6036040186882019, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.18975917078230178, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8267073631286621, + "proposal_oracle_utility_gain_over_base_at_8": -1.173292636871338, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.2231033444404602, + "selected_utility_gain_over_base_at_8": -1.7768966555595398, + "selector_regret_at_8": 0.6036040186882019, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.173292636871338, + "total_car_to_hidden_at_8": 1.7768966555595398 + }, + "1607773860/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.24177237819615016, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1665739511/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.007551252841949463, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.050940808914706356, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.8216220736503601, + "proposal_oracle_utility_gain_over_base_at_8": -1.17837792634964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.8140708208084106, + "selected_utility_gain_over_base_at_8": -1.1859291791915894, + "selector_regret_at_8": 0.007551252841949463, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.17837792634964, + "total_car_to_hidden_at_8": 1.1859291791915894 + }, + "1665739511/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1665739511/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2508232858898205, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1747178030/train0": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.15825371494561297, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8536934107542038, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8536934107542038, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1747178030/train1": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, + "branch_car_at_8": 0.027157030999660492, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.3089551740932353, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.11134227365255356, + "proposal_oracle_utility_gain_over_base_at_8": -0.034964315593242645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.08418524265289307, + "selected_utility_gain_over_base_at_8": -0.06212134659290314, + "selector_regret_at_8": 0.027157030999660492, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8886577263474464, + "total_car_to_hidden_at_8": 1.915814757347107 + }, + "1747178030/train2": { + "base_success": 0.0, + "base_utility": 0.1463065892457962, + "branch_car_at_8": 0.034092940390110016, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.08595818214364334, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12480536848306656, + "proposal_oracle_utility_gain_over_base_at_8": -0.021501220762729645, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09071242809295654, + "selected_utility_gain_over_base_at_8": -0.05559416115283966, + "selector_regret_at_8": 0.034092940390110016, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8751946315169334, + "total_car_to_hidden_at_8": 1.9092875719070435 + }, + "1881223807/train0": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.020236089825630188, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.019060791998604134, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0513521283864975, + "proposal_oracle_utility_gain_over_base_at_8": 0.0193251371383667, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03111603856086731, + "selected_utility_gain_over_base_at_8": -0.0009109526872634888, + "selector_regret_at_8": 0.020236089825630188, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9486478716135025, + "total_car_to_hidden_at_8": 1.9688839614391327 + }, + "1881223807/train1": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.009028241038322449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.13674585276395504, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.034886181354522705, + "proposal_oracle_utility_gain_over_base_at_8": 0.0028591901063919067, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.025857940316200256, + "selected_utility_gain_over_base_at_8": -0.006169050931930542, + "selector_regret_at_8": 0.009028241038322449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9651138186454773, + "total_car_to_hidden_at_8": 1.9741420596837997 + }, + "1881223807/train2": { + "base_success": 0.0, + "base_utility": 0.0320269912481308, + "branch_car_at_8": 0.024854734539985657, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2146835718150399, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.043482705950737, + "proposal_oracle_utility_gain_over_base_at_8": 0.011455714702606201, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.018627971410751343, + "selected_utility_gain_over_base_at_8": -0.013399019837379456, + "selector_regret_at_8": 0.024854734539985657, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.956517294049263, + "total_car_to_hidden_at_8": 1.9813720285892487 + }, + "1919198188/train0": { + "base_success": 0.0, + "base_utility": 0.037198856472969055, + "branch_car_at_8": 0.016854479908943176, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.15779226512372985, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08992795646190643, + "proposal_oracle_utility_gain_over_base_at_8": 0.05272909998893738, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07307347655296326, + "selected_utility_gain_over_base_at_8": 0.0358746200799942, + "selector_regret_at_8": 0.016854479908943176, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9100720435380936, + "total_car_to_hidden_at_8": 1.9269265234470367 + }, + "1919198188/train1": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.13114443223224567, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.046244338154792786, + "proposal_oracle_utility_gain_over_base_at_8": 0.009045466780662537, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046244338154792786, + "selected_utility_gain_over_base_at_8": 0.009045466780662537, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9537556618452072, + "total_car_to_hidden_at_8": 1.9537556618452072 + }, + "1919198188/train2": { + "base_success": 0.0, + "base_utility": 0.03719887137413025, + "branch_car_at_8": 0.018839523196220398, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.1669540555093941, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.06275103986263275, + "proposal_oracle_utility_gain_over_base_at_8": 0.025552168488502502, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.043911516666412354, + "selected_utility_gain_over_base_at_8": 0.0067126452922821045, + "selector_regret_at_8": 0.018839523196220398, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9372489601373672, + "total_car_to_hidden_at_8": 1.9560884833335876 + }, + "1991414747/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.05287821364799471, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1991414747/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9863876849412918, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.1306768270010748, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013612315058708191, + "selected_utility_gain_over_base_at_8": -1.9863876849412918, + "selector_regret_at_8": 1.9863876849412918, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9863876849412918 + }, + "1991414747/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.11896663871153935, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1997090716/train0": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 1.9691698104143143, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.4044963838715546, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.03083018958568573, + "selected_utility_gain_over_base_at_8": -0.10844460129737854, + "selector_regret_at_8": 1.9691698104143143, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9691698104143143 + }, + "1997090716/train1": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2554442550173647, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "1997090716/train2": { + "base_success": 0.0, + "base_utility": 0.13927479088306427, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.14617959762414967, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8607252091169357, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8607252091169357, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2015956957/train0": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, + "branch_car_at_8": 0.01770364921540022, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.18664994005668376, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.45124312303960323, + "proposal_oracle_utility_gain_over_base_at_8": 0.38412333317101, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.433539473824203, + "selected_utility_gain_over_base_at_8": 0.3664196839556098, + "selector_regret_at_8": 0.01770364921540022, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07863824404776096, + "total_car_to_hidden_at_8": 0.09634189326316119 + }, + "2015956957/train1": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, + "branch_car_at_8": 0.022195855714380743, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.16317219295797403, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05629209652543068, + "proposal_oracle_utility_gain_over_base_at_8": -0.010827693343162536, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.034096240811049935, + "selected_utility_gain_over_base_at_8": -0.033023549057543275, + "selector_regret_at_8": 0.022195855714380743, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.2, + "success_total_car_to_hidden_at_8": 0.2, + "support_gap_at_8": 0.4735892705619335, + "total_car_to_hidden_at_8": 0.49578512627631427 + }, + "2015956957/train2": { + "base_success": 0.0, + "base_utility": 0.06711978986859321, + "branch_car_at_8": 0.005573729798197746, + "hidden_chart_oracle_success_at_8": 0.2, + "hidden_chart_oracle_utility_at_8": 0.5298813670873642, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.1431252893441326, + "proposal_oracle_success_at_8": 0.2, + "proposal_oracle_success_gain_over_base_at_8": 0.2, + "proposal_oracle_utility_at_8": 0.43981083631515505, + "proposal_oracle_utility_gain_over_base_at_8": 0.3726910464465618, + "selected_success_at_8": 0.2, + "selected_success_gain_over_base_at_8": 0.2, + "selected_utility_at_8": 0.4342371065169573, + "selected_utility_gain_over_base_at_8": 0.36711731664836406, + "selector_regret_at_8": 0.005573729798197746, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.09007053077220917, + "total_car_to_hidden_at_8": 0.09564426057040691 + }, + "2054899126/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2054899126/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2054899126/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "211245842/train0": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, + "branch_car_at_8": 0.5908347219228745, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.16771630653845432, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8130142092704773, + "proposal_oracle_utility_gain_over_base_at_8": -0.02582019567489624, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22217948734760284, + "selected_utility_gain_over_base_at_8": -0.6166549175977707, + "selector_regret_at_8": 0.5908347219228745, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1869857907295227, + "total_car_to_hidden_at_8": 1.7778205126523972 + }, + "211245842/train1": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, + "branch_car_at_8": 0.6792904287576675, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07548939908426067, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8358442187309265, + "proposal_oracle_utility_gain_over_base_at_8": -0.0029901862144470215, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.15655378997325897, + "selected_utility_gain_over_base_at_8": -0.6822806149721146, + "selector_regret_at_8": 0.6792904287576675, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1641557812690735, + "total_car_to_hidden_at_8": 1.843446210026741 + }, + "211245842/train2": { + "base_success": 0.0, + "base_utility": 0.8388344049453735, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.13508468824583414, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1611655950546265, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1611655950546265, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "2141802816/train0": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 0.07543906569480896, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.05242798758366053, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.12472955882549286, + "proposal_oracle_utility_gain_over_base_at_8": 0.05392231047153473, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0492904931306839, + "selected_utility_gain_over_base_at_8": -0.02151675522327423, + "selector_regret_at_8": 0.07543906569480896, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.8752704411745071, + "total_car_to_hidden_at_8": 1.950709506869316 + }, + "2141802816/train1": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 0.019766509532928467, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07786951284417709, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.09589323401451111, + "proposal_oracle_utility_gain_over_base_at_8": 0.02508598566055298, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.07612672448158264, + "selected_utility_gain_over_base_at_8": 0.005319476127624512, + "selector_regret_at_8": 0.019766509532928467, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.904106765985489, + "total_car_to_hidden_at_8": 1.9238732755184174 + }, + "2141802816/train2": { + "base_success": 0.0, + "base_utility": 0.07080724835395813, + "branch_car_at_8": 1.861636370420456, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.09903289408743085, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9291927516460419, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13836362957954407, + "selected_utility_gain_over_base_at_8": 0.06755638122558594, + "selector_regret_at_8": 1.861636370420456, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.861636370420456 + }, + "256770874/train0": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, + "branch_car_at_8": 0.10820980370044708, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.030709639081801327, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.24107472598552704, + "proposal_oracle_utility_gain_over_base_at_8": -0.6146733313798904, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.13286492228507996, + "selected_utility_gain_over_base_at_8": -0.7228831350803375, + "selector_regret_at_8": 0.10820980370044708, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.758925274014473, + "total_car_to_hidden_at_8": 1.86713507771492 + }, + "256770874/train1": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, + "branch_car_at_8": 0.7044661641120911, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.03135306008748828, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8125, + "proposal_oracle_utility_gain_over_base_at_8": -0.04324805736541748, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.10803383588790894, + "selected_utility_gain_over_base_at_8": -0.7477142214775085, + "selector_regret_at_8": 0.7044661641120911, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1875, + "total_car_to_hidden_at_8": 1.891966164112091 + }, + "256770874/train2": { + "base_success": 0.0, + "base_utility": 0.8557480573654175, + "branch_car_at_8": 0.046216949820518494, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.046630940401554466, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.21061499416828156, + "proposal_oracle_utility_gain_over_base_at_8": -0.6451330631971359, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16439804434776306, + "selected_utility_gain_over_base_at_8": -0.6913500130176544, + "selector_regret_at_8": 0.046216949820518494, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7893850058317184, + "total_car_to_hidden_at_8": 1.835601955652237 + }, + "278008524/train0": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, + "branch_car_at_8": 0.0016456245793960989, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.11925245416072927, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.011432985425926745, + "proposal_oracle_utility_gain_over_base_at_8": -0.09603888296987861, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.009787360846530646, + "selected_utility_gain_over_base_at_8": -0.09768450754927471, + "selector_regret_at_8": 0.0016456245793960989, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.1023782094707713, + "total_car_to_hidden_at_8": 1.1040238340501674 + }, + "278008524/train1": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, + "branch_car_at_8": 0.011409573198761791, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.24605586147071373, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.03147180867381394, + "proposal_oracle_utility_gain_over_base_at_8": -0.07600005972199142, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.020062235475052148, + "selected_utility_gain_over_base_at_8": -0.08740963292075321, + "selector_regret_at_8": 0.011409573198761791, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.082339386222884, + "total_car_to_hidden_at_8": 1.0937489594216458 + }, + "278008524/train2": { + "base_success": 0.0, + "base_utility": 0.10747186839580536, + "branch_car_at_8": 0.020201382460072637, + "hidden_chart_oracle_success_at_8": 0.5, + "hidden_chart_oracle_utility_at_8": 1.113811194896698, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.15535141304867436, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032865648390725255, + "proposal_oracle_utility_gain_over_base_at_8": -0.0746062200050801, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012664265930652618, + "selected_utility_gain_over_base_at_8": -0.09480760246515274, + "selector_regret_at_8": 0.020201382460072637, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 1.0809455465059727, + "total_car_to_hidden_at_8": 1.1011469289660454 + }, + "416183106/train0": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, + "branch_car_at_8": 0.3360496461391449, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10129212810935397, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5778142213821411, + "proposal_oracle_utility_gain_over_base_at_8": 0.37748780846595764, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.24176457524299622, + "selected_utility_gain_over_base_at_8": 0.041438162326812744, + "selector_regret_at_8": 0.3360496461391449, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4221857786178589, + "total_car_to_hidden_at_8": 1.7582354247570038 + }, + "416183106/train1": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, + "branch_car_at_8": 1.7668711841106415, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07978206783428046, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.23312881588935852, + "selected_utility_gain_over_base_at_8": 0.03280240297317505, + "selector_regret_at_8": 1.7668711841106415, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7668711841106415 + }, + "416183106/train2": { + "base_success": 0.0, + "base_utility": 0.20032641291618347, + "branch_car_at_8": 1.812951534986496, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.18072825209943494, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7996735870838165, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.18704846501350403, + "selected_utility_gain_over_base_at_8": -0.013277947902679443, + "selector_regret_at_8": 1.812951534986496, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.812951534986496 + }, + "534717141/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9274959191679955, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.6159370955924921, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.07250408083200455, + "selected_utility_gain_over_base_at_8": -1.9274959191679955, + "selector_regret_at_8": 1.9274959191679955, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9274959191679955 + }, + "534717141/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 1.9558932781219482, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.14369576234023818, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.04410672187805176, + "selected_utility_gain_over_base_at_8": -1.9558932781219482, + "selector_regret_at_8": 1.9558932781219482, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9558932781219482 + }, + "534717141/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2870871533378466, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "551504635/train0": { + "base_success": 0.0, + "base_utility": 0.7239725589752197, + "branch_car_at_8": 0.033253490924835205, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.10932818664818449, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7250897884368896, + "proposal_oracle_utility_gain_over_base_at_8": 0.0011172294616699219, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.6918362975120544, + "selected_utility_gain_over_base_at_8": -0.03213626146316528, + "selector_regret_at_8": 0.033253490924835205, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2749102115631104, + "total_car_to_hidden_at_8": 1.3081637024879456 + }, + "551504635/train1": { + "base_success": 0.0, + "base_utility": 0.7247228622436523, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2299325778712667, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7224790453910828, + "proposal_oracle_utility_gain_over_base_at_8": -0.00224381685256958, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7224790453910828, + "selected_utility_gain_over_base_at_8": -0.00224381685256958, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.2775209546089172, + "total_car_to_hidden_at_8": 1.2775209546089172 + }, + "551504635/train2": { + "base_success": 0.0, + "base_utility": 0.7238867878913879, + "branch_car_at_8": 1.273692786693573, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2702494805053396, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.276113212108612, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.726307213306427, + "selected_utility_gain_over_base_at_8": 0.0024204254150390625, + "selector_regret_at_8": 1.273692786693573, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.273692786693573 + }, + "565369392/train0": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, + "branch_car_at_8": 0.06443698455890019, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.22308845281795123, + "proposal_oracle_success_at_8": 0.3333333333333333, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.7852714310089747, + "proposal_oracle_utility_gain_over_base_at_8": -0.014494215448697409, + "selected_success_at_8": 0.3333333333333333, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.7208344464500746, + "selected_utility_gain_over_base_at_8": -0.07893120000759761, + "selector_regret_at_8": 0.06443698455890019, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.3333333333333333, + "success_total_car_to_hidden_at_8": 0.3333333333333333, + "support_gap_at_8": 0.6413601487874985, + "total_car_to_hidden_at_8": 0.7057971333463987 + }, + "565369392/train1": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, + "branch_car_at_8": 0.046260833740234375, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.13991651305345584, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.17366480827331543, + "proposal_oracle_utility_gain_over_base_at_8": -0.6261008381843567, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.12740397453308105, + "selected_utility_gain_over_base_at_8": -0.6723616719245911, + "selector_regret_at_8": 0.046260833740234375, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2529667715231578, + "total_car_to_hidden_at_8": 1.2992276052633922 + }, + "565369392/train2": { + "base_success": 0.3333333333333333, + "base_utility": 0.7997656464576721, + "branch_car_at_8": 0.04557279869914055, + "hidden_chart_oracle_success_at_8": 0.6666666666666666, + "hidden_chart_oracle_utility_at_8": 1.4266315797964733, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.17390197648626904, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.3333333333333333, + "proposal_oracle_utility_at_8": 0.1326704372962316, + "proposal_oracle_utility_gain_over_base_at_8": -0.6670952091614405, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.3333333333333333, + "selected_utility_at_8": 0.08709763859709103, + "selected_utility_gain_over_base_at_8": -0.712668007860581, + "selector_regret_at_8": 0.04557279869914055, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.6666666666666666, + "success_total_car_to_hidden_at_8": 0.6666666666666666, + "support_gap_at_8": 1.2939611425002415, + "total_car_to_hidden_at_8": 1.339533941199382 + }, + "565621826/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "565621826/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "565621826/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.0, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.0, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "585904130/train0": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, + "branch_car_at_8": 0.043168901621053614, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, + "outcome_ptr_at_8": 0.3333333333333333, + "pairwise_causal_calibration_ece": 0.10391809443539522, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.05598785417775313, + "proposal_oracle_utility_gain_over_base_at_8": -0.019304243847727776, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.012818952556699514, + "selected_utility_gain_over_base_at_8": -0.06247314546878139, + "selector_regret_at_8": 0.043168901621053614, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1816591378301382, + "total_car_to_hidden_at_8": 0.2248280394511918 + }, + "585904130/train1": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, + "branch_car_at_8": 0.012068324722349644, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07276320034061332, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.032918572425842285, + "proposal_oracle_utility_gain_over_base_at_8": -0.04237352559963862, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.02085024770349264, + "selected_utility_gain_over_base_at_8": -0.05444185032198826, + "selector_regret_at_8": 0.012068324722349644, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.20472841958204904, + "total_car_to_hidden_at_8": 0.2167967443043987 + }, + "585904130/train2": { + "base_success": 0.0, + "base_utility": 0.07529209802548091, + "branch_car_at_8": 0.011032561771571636, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23764699200789133, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.2625659187813512, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07848061621189117, + "proposal_oracle_utility_gain_over_base_at_8": 0.0031885181864102683, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.06744805444031954, + "selected_utility_gain_over_base_at_8": -0.007844043585161367, + "selector_regret_at_8": 0.011032561771571636, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.15916637579600015, + "total_car_to_hidden_at_8": 0.1701989375675718 + }, + "626486813/train0": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, + "branch_car_at_8": 0.018570441869087517, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.16162412565990822, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5615248780231923, + "proposal_oracle_utility_gain_over_base_at_8": 0.0560848613968119, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5429544361541048, + "selected_utility_gain_over_base_at_8": 0.037514419527724385, + "selector_regret_at_8": 0.018570441869087517, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0801479306537658, + "total_car_to_hidden_at_8": 0.08356922247912735 + }, + "626486813/train1": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, + "branch_car_at_8": 0.02500777377281338, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.2420589895144555, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07365635968744755, + "proposal_oracle_utility_gain_over_base_at_8": -0.43178365693893284, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.04864858591463417, + "selected_utility_gain_over_base_at_8": -0.4567914307117462, + "selector_regret_at_8": 0.02500777377281338, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5414478182792664, + "total_car_to_hidden_at_8": 0.5638198243686929 + }, + "626486813/train2": { + "base_success": 0.25, + "base_utility": 0.5054400166263804, + "branch_car_at_8": 0.031722720130346715, + "hidden_chart_oracle_success_at_8": 0.25, + "hidden_chart_oracle_utility_at_8": 0.6124684102833271, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.23561688988950158, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.07003506645560265, + "proposal_oracle_utility_gain_over_base_at_8": -0.43540495017077774, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.03831234632525593, + "selected_utility_gain_over_base_at_8": -0.46712767030112445, + "selector_regret_at_8": 0.031722720130346715, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.25, + "success_total_car_to_hidden_at_8": 0.25, + "support_gap_at_8": 0.5432130172848701, + "total_car_to_hidden_at_8": 0.5741560639580712 + }, + "636680137/train0": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.1699944666253151, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.012414653785526752, + "proposal_oracle_utility_gain_over_base_at_8": -1.9875853462144732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.012414653785526752, + "selected_utility_gain_over_base_at_8": -1.9875853462144732, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9875853462144732, + "total_car_to_hidden_at_8": 1.9875853462144732 + }, + "636680137/train1": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.07628525991805817, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.013928472995758057, + "proposal_oracle_utility_gain_over_base_at_8": -1.986071527004242, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.013928472995758057, + "selected_utility_gain_over_base_at_8": -1.986071527004242, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.986071527004242, + "total_car_to_hidden_at_8": 1.986071527004242 + }, + "636680137/train2": { + "base_success": 1.0, + "base_utility": 2.0, + "branch_car_at_8": 0.006491700187325478, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.12019361687075947, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -1.0, + "proposal_oracle_utility_at_8": 0.02436021901667118, + "proposal_oracle_utility_gain_over_base_at_8": -1.9756397809833288, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -1.0, + "selected_utility_at_8": 0.017868518829345703, + "selected_utility_gain_over_base_at_8": -1.9821314811706543, + "selector_regret_at_8": 0.006491700187325478, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9756397809833288, + "total_car_to_hidden_at_8": 1.9821314811706543 + }, + "706479600/train0": { + "base_success": 0.0, + "base_utility": 0.8125406503677368, + "branch_car_at_8": 1.1865743398666382, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0922387871049479, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1874593496322632, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.8134256601333618, + "selected_utility_gain_over_base_at_8": 0.000885009765625, + "selector_regret_at_8": 1.1865743398666382, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.1865743398666382 + }, + "706479600/train1": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.16443847051324711, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "706479600/train2": { + "base_success": 0.0, + "base_utility": 0.8350887894630432, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.22468613298280815, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.1649112105369568, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.1649112105369568, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "750448296/train0": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, + "branch_car_at_8": 0.07732226233929396, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.14360585133877676, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.1069931760430336, + "proposal_oracle_utility_gain_over_base_at_8": -0.004540383815765381, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.029670913703739643, + "selected_utility_gain_over_base_at_8": -0.08186264615505934, + "selector_regret_at_8": 0.07732226233929396, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.12881441414356232, + "total_car_to_hidden_at_8": 0.20613667648285627 + }, + "750448296/train1": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, + "branch_car_at_8": 0.02701035887002945, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.2344413298276417, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.07342664152383804, + "proposal_oracle_utility_gain_over_base_at_8": -0.03810691833496094, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.046416282653808594, + "selected_utility_gain_over_base_at_8": -0.06511727720499039, + "selector_regret_at_8": 0.02701035887002945, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.16238094866275787, + "total_car_to_hidden_at_8": 0.18939130753278732 + }, + "750448296/train2": { + "base_success": 0.0, + "base_utility": 0.11153355985879898, + "branch_car_at_8": 0.01770863775163889, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.23580759018659592, + "outcome_ptr_at_8": 0.0, + "pairwise_causal_calibration_ece": 0.14149358436045273, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.0669428389519453, + "proposal_oracle_utility_gain_over_base_at_8": -0.044590720906853676, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.049234201200306416, + "selected_utility_gain_over_base_at_8": -0.062299358658492565, + "selector_regret_at_8": 0.01770863775163889, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.1688647512346506, + "total_car_to_hidden_at_8": 0.1865733889862895 + }, + "75070641/train0": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, + "branch_car_at_8": 0.07482179999351501, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.2497251397544613, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.2396746575832367, + "proposal_oracle_utility_gain_over_base_at_8": 0.03692302107810974, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.16485285758972168, + "selected_utility_gain_over_base_at_8": -0.03789877891540527, + "selector_regret_at_8": 0.07482179999351501, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.7603253424167633, + "total_car_to_hidden_at_8": 1.8351471424102783 + }, + "75070641/train1": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, + "branch_car_at_8": 0.5863077491521835, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.18559462278331035, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.8128874897956848, + "proposal_oracle_utility_gain_over_base_at_8": 0.6101358532905579, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22657974064350128, + "selected_utility_gain_over_base_at_8": 0.02382810413837433, + "selector_regret_at_8": 0.5863077491521835, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.1871125102043152, + "total_car_to_hidden_at_8": 1.7734202593564987 + }, + "75070641/train2": { + "base_success": 0.0, + "base_utility": 0.20275163650512695, + "branch_car_at_8": 0.3710349500179291, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.09051828052202912, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5650836229324341, + "proposal_oracle_utility_gain_over_base_at_8": 0.36233198642730713, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.194048672914505, + "selected_utility_gain_over_base_at_8": -0.008702963590621948, + "selector_regret_at_8": 0.3710349500179291, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.434916377067566, + "total_car_to_hidden_at_8": 1.805951327085495 + }, + "767747424/train0": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, + "branch_car_at_8": 0.015960504300892353, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, + "outcome_ptr_at_8": 0.25, + "pairwise_causal_calibration_ece": 0.21088094062197144, + "proposal_oracle_success_at_8": 0.25, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5703969299793243, + "proposal_oracle_utility_gain_over_base_at_8": 0.022872742265462875, + "selected_success_at_8": 0.25, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.554436425678432, + "selected_utility_gain_over_base_at_8": 0.006912237964570522, + "selector_regret_at_8": 0.015960504300892353, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.05905248410999775, + "total_car_to_hidden_at_8": 0.0750129884108901 + }, + "767747424/train1": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, + "branch_car_at_8": 0.04561196081340313, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.07531396139335009, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11342880013398826, + "proposal_oracle_utility_gain_over_base_at_8": -0.4340953875798732, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.06781683932058513, + "selected_utility_gain_over_base_at_8": -0.47970734839327633, + "selector_regret_at_8": 0.04561196081340313, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.08526064571924508, + "total_car_to_hidden_at_8": 0.12081881728954613 + }, + "767747424/train2": { + "base_success": 0.25, + "base_utility": 0.5475241877138615, + "branch_car_at_8": 0.024658456211909652, + "hidden_chart_oracle_success_at_8": 0.0, + "hidden_chart_oracle_utility_at_8": 0.18863565661013126, + "outcome_ptr_at_8": 0.75, + "pairwise_causal_calibration_ece": 0.14269822176062383, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": -0.25, + "proposal_oracle_utility_at_8": 0.11239026742987335, + "proposal_oracle_utility_gain_over_base_at_8": -0.4351339202839881, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.25, + "selected_utility_at_8": 0.0877318112179637, + "selected_utility_gain_over_base_at_8": -0.45979237649589777, + "selector_regret_at_8": 0.024658456211909652, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.07677002181299031, + "total_car_to_hidden_at_8": 0.10090384539216757 + }, + "77147264/train0": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, + "branch_car_at_8": 1.868594542145729, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.06512707818376468, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": -0.5, + "selected_utility_at_8": 0.13140545785427094, + "selected_utility_gain_over_base_at_8": -1.2901740223169327, + "selector_regret_at_8": 1.868594542145729, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.868594542145729 + }, + "77147264/train1": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.2630803856693186, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 1.426636129617691, + "proposal_oracle_utility_gain_over_base_at_8": 0.005056649446487427, + "selected_success_at_8": 0.5, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 1.426636129617691, + "selected_utility_gain_over_base_at_8": 0.005056649446487427, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.5, + "support_gap_at_8": 0.573363870382309, + "total_car_to_hidden_at_8": 0.573363870382309 + }, + "77147264/train2": { + "base_success": 0.5, + "base_utility": 1.4215794801712036, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.12309184879211332, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 0.5, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 0.5784205198287964, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 0.5, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 0.5784205198287964, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "794181626/train0": { + "base_success": 0.0, + "base_utility": 0.1733371466398239, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.12119924562800825, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.826662853360176, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.826662853360176, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "794181626/train1": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28339362113630795, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "794181626/train2": { + "base_success": 0.0, + "base_utility": 0.18095605075359344, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.0, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.8190439492464066, + "selected_success_at_8": 1.0, + "selected_success_gain_over_base_at_8": 1.0, + "selected_utility_at_8": 2.0, + "selected_utility_gain_over_base_at_8": 1.8190439492464066, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 0.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 0.0 + }, + "957101489/train0": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, + "branch_car_at_8": 0.0, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.20250855048512742, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.5881698727607727, + "proposal_oracle_utility_gain_over_base_at_8": 0.3842049241065979, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.5881698727607727, + "selected_utility_gain_over_base_at_8": 0.3842049241065979, + "selector_regret_at_8": 0.0, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.4118301272392273, + "total_car_to_hidden_at_8": 1.4118301272392273 + }, + "957101489/train1": { + "base_success": 0.0, + "base_utility": 0.20372186601161957, + "branch_car_at_8": 1.7757539749145508, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.07643021229779011, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7962781339883804, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.22424602508544922, + "selected_utility_gain_over_base_at_8": 0.02052415907382965, + "selector_regret_at_8": 1.7757539749145508, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.7757539749145508 + }, + "957101489/train2": { + "base_success": 0.0, + "base_utility": 0.2039649486541748, + "branch_car_at_8": 1.432861328125, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.21219243624964876, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.7960350513458252, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.567138671875, + "selected_utility_gain_over_base_at_8": 0.3631737232208252, + "selector_regret_at_8": 1.432861328125, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.432861328125 + }, + "998761162/train0": { + "base_success": 0.0, + "base_utility": 0.0689050555229187, + "branch_car_at_8": 0.07749588787555695, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.23606164078349207, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.08403940498828888, + "proposal_oracle_utility_gain_over_base_at_8": 0.015134349465370178, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.006543517112731934, + "selected_utility_gain_over_base_at_8": -0.06236153841018677, + "selector_regret_at_8": 0.07749588787555695, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.9159605950117111, + "total_car_to_hidden_at_8": 1.993456482887268 + }, + "998761162/train1": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, + "branch_car_at_8": 1.9512543678283691, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.28991947553571784, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.04874563217163086, + "selected_utility_gain_over_base_at_8": -0.019070938229560852, + "selector_regret_at_8": 1.9512543678283691, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.9512543678283691 + }, + "998761162/train2": { + "base_success": 0.0, + "base_utility": 0.06781657040119171, + "branch_car_at_8": 1.939187914133072, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 1.0, + "pairwise_causal_calibration_ece": 0.11706618974004251, + "proposal_oracle_success_at_8": 1.0, + "proposal_oracle_success_gain_over_base_at_8": 1.0, + "proposal_oracle_utility_at_8": 2.0, + "proposal_oracle_utility_gain_over_base_at_8": 1.9321834295988083, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.0608120858669281, + "selected_utility_gain_over_base_at_8": -0.007004484534263611, + "selector_regret_at_8": 1.939187914133072, + "success_selector_gap_at_8": 1.0, + "success_support_gap_at_8": 0.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 0.0, + "total_car_to_hidden_at_8": 1.939187914133072 + } +} diff --git a/workspace/runs/ctt_val_rollout_comparison/metrics_by_task.json b/workspace/runs/ctt_val_rollout_comparison/metrics_by_task.json new file mode 100644 index 0000000000000000000000000000000000000000..54ab0b7dda30c9a1c5314aec27032079a5f8586c --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/metrics_by_task.json @@ -0,0 +1,117 @@ +{ + "LiftPegUpright-v1": { + "base_success": 0.3333333333333333, + "base_utility": 1.0669810656044219, + "branch_car_at_8": 0.516516399052408, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.5, + "pairwise_causal_calibration_ece": 0.1608032528409541, + "proposal_oracle_success_at_8": 0.5, + "proposal_oracle_success_gain_over_base_at_8": 0.16666666666666666, + "proposal_oracle_utility_at_8": 1.3387502448426352, + "proposal_oracle_utility_gain_over_base_at_8": 0.2717691792382134, + "selected_success_at_8": 0.2222222222222222, + "selected_success_gain_over_base_at_8": -0.1111111111111111, + "selected_utility_at_8": 0.8222338457902273, + "selected_utility_gain_over_base_at_8": -0.24474721981419456, + "selector_regret_at_8": 0.516516399052408, + "success_selector_gap_at_8": 0.2777777777777778, + "success_support_gap_at_8": 0.5, + "success_total_car_to_hidden_at_8": 0.7777777777777778, + "support_gap_at_8": 0.6612497551573647, + "total_car_to_hidden_at_8": 1.1777661542097728 + }, + "PickCube-v1": { + "base_success": 0.0, + "base_utility": 0.15342965722084045, + "branch_car_at_8": 0.2721092589199543, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6666666666666666, + "pairwise_causal_calibration_ece": 0.17559224662436113, + "proposal_oracle_success_at_8": 0.0, + "proposal_oracle_success_gain_over_base_at_8": 0.0, + "proposal_oracle_utility_at_8": 0.3654347012440364, + "proposal_oracle_utility_gain_over_base_at_8": 0.2120050440231959, + "selected_success_at_8": 0.0, + "selected_success_gain_over_base_at_8": 0.0, + "selected_utility_at_8": 0.09332544232408206, + "selected_utility_gain_over_base_at_8": -0.0601042148967584, + "selector_regret_at_8": 0.2721092589199543, + "success_selector_gap_at_8": 0.0, + "success_support_gap_at_8": 1.0, + "success_total_car_to_hidden_at_8": 1.0, + "support_gap_at_8": 1.6345652987559636, + "total_car_to_hidden_at_8": 1.9066745576759179 + }, + "PullCube-v1": { + "base_success": 0.29411764705882354, + "base_utility": 0.6512299207833541, + "branch_car_at_8": 0.05991193203954026, + "hidden_chart_oracle_success_at_8": 0.3235294117647059, + "hidden_chart_oracle_utility_at_8": 0.7823218367993832, + "outcome_ptr_at_8": 0.3235294117647059, + "pairwise_causal_calibration_ece": 0.1645321267392569, + "proposal_oracle_success_at_8": 0.18627450980392157, + "proposal_oracle_success_gain_over_base_at_8": -0.10784313725490197, + "proposal_oracle_utility_at_8": 0.4324846717342963, + "proposal_oracle_utility_gain_over_base_at_8": -0.21874524904905782, + "selected_success_at_8": 0.16666666666666666, + "selected_success_gain_over_base_at_8": -0.12745098039215685, + "selected_utility_at_8": 0.372572739694756, + "selected_utility_gain_over_base_at_8": -0.2786571810885981, + "selector_regret_at_8": 0.05991193203954026, + "success_selector_gap_at_8": 0.0196078431372549, + "success_support_gap_at_8": 0.14705882352941177, + "success_total_car_to_hidden_at_8": 0.16666666666666666, + "support_gap_at_8": 0.3689149604584136, + "total_car_to_hidden_at_8": 0.42758709733321915 + }, + "PushCube-v1": { + "base_success": 0.358974358974359, + "base_utility": 0.7709262684369699, + "branch_car_at_8": 0.4573507461792383, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6153846153846154, + "pairwise_causal_calibration_ece": 0.16409670072723917, + "proposal_oracle_success_at_8": 0.7692307692307693, + "proposal_oracle_success_gain_over_base_at_8": 0.41025641025641024, + "proposal_oracle_utility_at_8": 1.5530982193274376, + "proposal_oracle_utility_gain_over_base_at_8": 0.7821719508904678, + "selected_success_at_8": 0.5384615384615384, + "selected_success_gain_over_base_at_8": 0.1794871794871795, + "selected_utility_at_8": 1.0957474731481993, + "selected_utility_gain_over_base_at_8": 0.3248212047112294, + "selector_regret_at_8": 0.4573507461792383, + "success_selector_gap_at_8": 0.23076923076923078, + "success_support_gap_at_8": 0.23076923076923078, + "success_total_car_to_hidden_at_8": 0.46153846153846156, + "support_gap_at_8": 0.4469017806725624, + "total_car_to_hidden_at_8": 0.9042525268518008 + }, + "StackCube-v1": { + "base_success": 0.2, + "base_utility": 0.8034174028370116, + "branch_car_at_8": 0.593565668993526, + "hidden_chart_oracle_success_at_8": 1.0, + "hidden_chart_oracle_utility_at_8": 2.0, + "outcome_ptr_at_8": 0.6, + "pairwise_causal_calibration_ece": 0.1529426346924437, + "proposal_oracle_success_at_8": 0.4444444444444444, + "proposal_oracle_success_gain_over_base_at_8": 0.24444444444444444, + "proposal_oracle_utility_at_8": 1.1668234914541245, + "proposal_oracle_utility_gain_over_base_at_8": 0.3634060886171129, + "selected_success_at_8": 0.17777777777777778, + "selected_success_gain_over_base_at_8": -0.022222222222222223, + "selected_utility_at_8": 0.5732578224605984, + "selected_utility_gain_over_base_at_8": -0.23015958037641313, + "selector_regret_at_8": 0.593565668993526, + "success_selector_gap_at_8": 0.26666666666666666, + "success_support_gap_at_8": 0.5555555555555556, + "success_total_car_to_hidden_at_8": 0.8222222222222222, + "support_gap_at_8": 0.8331765085458755, + "total_car_to_hidden_at_8": 1.4267421775394016 + } +} diff --git a/workspace/runs/ctt_val_rollout_comparison/report.md b/workspace/runs/ctt_val_rollout_comparison/report.md index f0b158ffbe194d9362a411297fc7cdd475be5df7..48849cb5d7003c231f46cb163de4f07aa4da4e0d 100644 --- a/workspace/runs/ctt_val_rollout_comparison/report.md +++ b/workspace/runs/ctt_val_rollout_comparison/report.md @@ -6,11 +6,42 @@ K: `8` | Metric | N | Micro mean | 95% CI | | --- | ---: | ---: | ---: | -| branch_car_at_8 | 207 | 0.2936 | [0.2131, 0.3721] | -| outcome_ptr_at_8 | 207 | 0.4589 | [0.3913, 0.5266] | -| pairwise_causal_calibration_ece | 207 | 0.1618 | [0.1463, 0.1771] | -| selector_regret_at_8 | 207 | 0.2936 | [0.2131, 0.3721] | -| support_gap_at_8 | 207 | 0.5283 | [0.4234, 0.6341] | +| base_success | 207 | 0.2850 | [0.2271, 0.3478] | +| base_utility | 207 | 0.7358 | [0.6279, 0.8512] | +| branch_car_at_8 | 207 | 0.2936 | [0.2100, 0.3738] | +| hidden_chart_oracle_success_at_8 | 207 | 0.6667 | [0.6087, 0.7246] | +| hidden_chart_oracle_utility_at_8 | 207 | 1.4000 | [1.2940, 1.5057] | +| outcome_ptr_at_8 | 207 | 0.4589 | [0.3913, 0.5314] | +| pairwise_causal_calibration_ece | 207 | 0.1618 | [0.1468, 0.1771] | +| proposal_oracle_success_at_8 | 207 | 0.3768 | [0.3092, 0.4444] | +| proposal_oracle_success_gain_over_base_at_8 | 207 | 0.0918 | [0.0193, 0.1594] | +| proposal_oracle_utility_at_8 | 207 | 0.8811 | [0.7586, 1.0042] | +| proposal_oracle_utility_gain_over_base_at_8 | 207 | 0.1453 | [0.0316, 0.2667] | +| selected_success_at_8 | 207 | 0.2415 | [0.1884, 0.3043] | +| selected_success_gain_over_base_at_8 | 207 | -0.0435 | [-0.1111, 0.0242] | +| selected_utility_at_8 | 207 | 0.5875 | [0.4854, 0.7024] | +| selected_utility_gain_over_base_at_8 | 207 | -0.1483 | [-0.2654, -0.0237] | +| selector_regret_at_8 | 207 | 0.2936 | [0.2100, 0.3738] | +| success_selector_gap_at_8 | 207 | 0.1353 | [0.0870, 0.1884] | +| success_support_gap_at_8 | 207 | 0.2947 | [0.2319, 0.3623] | +| success_total_car_to_hidden_at_8 | 207 | 0.4300 | [0.3575, 0.4976] | +| support_gap_at_8 | 207 | 0.5283 | [0.4234, 0.6411] | +| total_car_to_hidden_at_8 | 207 | 0.8213 | [0.6955, 0.9329] | + +| Success/Utility | Mean | +| --- | ---: | +| base_success_rate | 0.2850 | +| selected_success_rate | 0.2415 | +| proposal_oracle_success_rate | 0.3768 | +| hidden_chart_oracle_success_rate | 0.6667 | +| selected_success_gain_over_base | -0.0435 | +| proposal_oracle_success_gain_over_base | 0.0918 | +| success_support_gap | 0.2947 | +| success_selector_gap | 0.1353 | +| base_utility_mean | 0.7358 | +| selected_utility_mean | 0.5875 | +| proposal_oracle_utility_mean | 0.8811 | +| hidden_chart_oracle_utility_mean | 1.4000 | These are measured generated-candidate rollouts, not PPTC proxies. diff --git a/workspace/runs/ctt_val_rollout_comparison/source_data_hash.txt b/workspace/runs/ctt_val_rollout_comparison/source_data_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..d6cc02ca1808e0515839939b88beab0acaad5cbc --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/source_data_hash.txt @@ -0,0 +1 @@ +90b431047a1f35cec095cbafa7de5bc0126cb0d0f4db3ff31843dca369977957 diff --git a/workspace/runs/ctt_val_rollout_comparison/split_hash.txt b/workspace/runs/ctt_val_rollout_comparison/split_hash.txt new file mode 100644 index 0000000000000000000000000000000000000000..8533e80f235606cef4ba0119a3ac3806d63a0983 --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/split_hash.txt @@ -0,0 +1 @@ +9d1dc1c10868f76fa080895df0f2730c173f12b6db14932599d55c2b60859381 diff --git a/workspace/runs/ctt_val_rollout_comparison/table.tex b/workspace/runs/ctt_val_rollout_comparison/table.tex index 31750a01692c07356f768a96c9ae4da98e65563e..40a7939a99f88e3c4661ccb20fbde320c103f133 100644 --- a/workspace/runs/ctt_val_rollout_comparison/table.tex +++ b/workspace/runs/ctt_val_rollout_comparison/table.tex @@ -3,10 +3,26 @@ \toprule Metric & N & Micro mean & CI low & CI high \\ \midrule -branch\_car\_at\_8 & 207 & 0.2936 & 0.2131 & 0.3721 \\ -outcome\_ptr\_at\_8 & 207 & 0.4589 & 0.3913 & 0.5266 \\ -pairwise\_causal\_calibration\_ece & 207 & 0.1618 & 0.1463 & 0.1771 \\ -selector\_regret\_at\_8 & 207 & 0.2936 & 0.2131 & 0.3721 \\ -support\_gap\_at\_8 & 207 & 0.5283 & 0.4234 & 0.6341 \\ +base\_success & 207 & 0.2850 & 0.2271 & 0.3478 \\ +base\_utility & 207 & 0.7358 & 0.6279 & 0.8512 \\ +branch\_car\_at\_8 & 207 & 0.2936 & 0.2100 & 0.3738 \\ +hidden\_chart\_oracle\_success\_at\_8 & 207 & 0.6667 & 0.6087 & 0.7246 \\ +hidden\_chart\_oracle\_utility\_at\_8 & 207 & 1.4000 & 1.2940 & 1.5057 \\ +outcome\_ptr\_at\_8 & 207 & 0.4589 & 0.3913 & 0.5314 \\ +pairwise\_causal\_calibration\_ece & 207 & 0.1618 & 0.1468 & 0.1771 \\ +proposal\_oracle\_success\_at\_8 & 207 & 0.3768 & 0.3092 & 0.4444 \\ +proposal\_oracle\_success\_gain\_over\_base\_at\_8 & 207 & 0.0918 & 0.0193 & 0.1594 \\ +proposal\_oracle\_utility\_at\_8 & 207 & 0.8811 & 0.7586 & 1.0042 \\ +proposal\_oracle\_utility\_gain\_over\_base\_at\_8 & 207 & 0.1453 & 0.0316 & 0.2667 \\ +selected\_success\_at\_8 & 207 & 0.2415 & 0.1884 & 0.3043 \\ +selected\_success\_gain\_over\_base\_at\_8 & 207 & -0.0435 & -0.1111 & 0.0242 \\ +selected\_utility\_at\_8 & 207 & 0.5875 & 0.4854 & 0.7024 \\ +selected\_utility\_gain\_over\_base\_at\_8 & 207 & -0.1483 & -0.2654 & -0.0237 \\ +selector\_regret\_at\_8 & 207 & 0.2936 & 0.2100 & 0.3738 \\ +success\_selector\_gap\_at\_8 & 207 & 0.1353 & 0.0870 & 0.1884 \\ +success\_support\_gap\_at\_8 & 207 & 0.2947 & 0.2319 & 0.3623 \\ +success\_total\_car\_to\_hidden\_at\_8 & 207 & 0.4300 & 0.3575 & 0.4976 \\ +support\_gap\_at\_8 & 207 & 0.5283 & 0.4234 & 0.6411 \\ +total\_car\_to\_hidden\_at\_8 & 207 & 0.8213 & 0.6955 & 0.9329 \\ \bottomrule \end{tabular} diff --git a/workspace/runs/ctt_val_rollout_comparison/train.log b/workspace/runs/ctt_val_rollout_comparison/train.log new file mode 100644 index 0000000000000000000000000000000000000000..518e1c04f4cca9e59f377237f5a12c0cf039c704 --- /dev/null +++ b/workspace/runs/ctt_val_rollout_comparison/train.log @@ -0,0 +1 @@ +comparison artifact; source runs trained/evaluated separately diff --git a/workspace/runs/summary_ctt.csv b/workspace/runs/summary_ctt.csv index 66d8de6e3b8412257dc9df42985bb7ce92805234..34a48375c06bee2bee303dd0078d880ce380cc54 100644 --- a/workspace/runs/summary_ctt.csv +++ b/workspace/runs/summary_ctt.csv @@ -1,25 +1,36 @@ -method,run_path,clean,status,success,proposal_oracle,support_gap,selector_gap,outcome_ptr,pptc_0p20,pptc_0p40,negative_near_0p20,calibration_ece,seeds,ci -V0 residual transport diagnostic,runs/reproduce_v0,yes,measured diagnostic baseline,0.3890,0.4435,0.1264,0.0545,n/a,0.2366,0.5269,0.0533,n/a,3,see run -CTT gated residual,runs/ctt_gated_residual_full_seed0,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT gated residual,runs/ctt_gated_residual_full_seed0_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.2319,0.5797,0.0512,n/a,47,see run -CTT gated residual,runs/ctt_gated_residual_full_seed1,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT gated residual,runs/ctt_gated_residual_full_seed1_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.2174,0.6232,0.0644,n/a,47,see run -CTT gated residual,runs/ctt_gated_residual_full_seed2,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT gated residual,runs/ctt_gated_residual_full_seed2_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.2464,0.6377,0.0424,n/a,47,see run -CTT gated residual,runs/ctt_gated_residual_smoke,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT gated residual,runs/ctt_gated_residual_smoke_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.5000,0.7500,0.0573,n/a,unknown,see run -CTT gated residual,runs/ctt_gated_residual_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.1739,0.5362,0.0281,n/a,47,see run -CTT residual train,runs/ctt_residual_full_seed0,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT residual proxy,runs/ctt_residual_full_seed0_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.2029,0.5797,0.0289,n/a,47,see run -CTT residual train,runs/ctt_residual_full_seed1,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT residual proxy,runs/ctt_residual_full_seed1_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.2319,0.5942,0.0410,n/a,47,see run -CTT residual train,runs/ctt_residual_full_seed2,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT residual proxy,runs/ctt_residual_full_seed2_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.1594,0.6522,0.0189,n/a,47,see run -CTT residual train,runs/ctt_residual_smoke,yes,train,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run -CTT residual proxy,runs/ctt_residual_smoke_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,1.0000,1.0000,0.1010,n/a,unknown,see run -CTT residual proxy,runs/ctt_residual_val_proxy,yes,proxy,n/a,n/a,n/a,n/a,n/a,0.3478,0.7101,0.0397,n/a,47,see run -task memory,runs/task_memory_val_proxy,yes,proxy baseline,n/a,n/a,n/a,n/a,n/a,0.3188,0.5362,0.0175,n/a,47,see run -local atlas,runs/local_atlas_val_proxy,yes,proxy baseline,n/a,n/a,n/a,n/a,n/a,0.4058,0.6812,0.0368,n/a,47,see run -ctt residual full,runs/ctt_val_proxy_comparison,yes,proxy aggregate gate=True,n/a,n/a,n/a,n/a,n/a,0.1981,0.6087,0.0296,n/a,3,aggregate over run means -ctt gated residual full,runs/ctt_val_proxy_comparison,yes,proxy aggregate gate=False,n/a,n/a,n/a,n/a,n/a,0.2319,0.6135,0.0527,n/a,3,aggregate over run means -Utility energy smoke,runs/utility_energy_smoke,train-only,calibration diagnostic,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.1316,train split,see run +method,run_path,clean,status,gate,base_success,success,proposal_oracle,hidden_oracle,support_gap,selector_gap,success_support_gap,success_selector_gap,outcome_ptr,pptc_0p20,pptc_0p40,negative_near_0p20,calibration_ece,seeds,ci +V0 residual transport diagnostic,runs/reproduce_v0,yes,measured diagnostic baseline,legacy diagnostic,0.2974,0.3890,0.4435,n/a,0.1264,0.0545,0.1264,0.0545,n/a,0.2366,0.5269,0.0533,n/a,3,see run +CTT gated residual,runs/ctt_gated_residual_full_seed0,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT gated residual,runs/ctt_gated_residual_full_seed0_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.2319,0.5797,0.0512,n/a,47,see run +CTT gated residual,runs/ctt_gated_residual_full_seed1,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT gated residual,runs/ctt_gated_residual_full_seed1_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.2174,0.6232,0.0644,n/a,47,see run +CTT gated residual,runs/ctt_gated_residual_full_seed2,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT gated residual,runs/ctt_gated_residual_full_seed2_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.2464,0.6377,0.0424,n/a,47,see run +CTT gated residual,runs/ctt_gated_residual_smoke,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT gated residual,runs/ctt_gated_residual_smoke_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.5000,0.7500,0.0573,n/a,unknown,see run +CTT gated residual,runs/ctt_gated_residual_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.1739,0.5362,0.0281,n/a,47,see run +CTT residual train,runs/ctt_residual_full_seed0,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT residual proxy,runs/ctt_residual_full_seed0_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.2029,0.5797,0.0289,n/a,47,see run +CTT residual train,runs/ctt_residual_full_seed1,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT residual proxy,runs/ctt_residual_full_seed1_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.2319,0.5942,0.0410,n/a,47,see run +CTT residual train,runs/ctt_residual_full_seed2,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT residual proxy,runs/ctt_residual_full_seed2_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.1594,0.6522,0.0189,n/a,47,see run +CTT residual validation measured,runs/ctt_residual_rollout_direct_smoke_seed0_v3,yes,measured rollout K=2,"fail:selected<47.45,oracle<50,OutcomePTR<=V0oracle,seeds<3",0.0000,0.0000,0.0000,0.0000,0.1798,0.0000,0.0000,0.0000,0.0000,n/a,n/a,n/a,0.4932,1,see measured_metrics +CTT residual test measured,runs/ctt_residual_rollout_test_seed0,yes,measured rollout K=8,"fail:selected<47.45,support_gap>7,selector_gap>3",0.2917,0.3125,0.5208,0.7292,0.4438,0.4304,0.2500,0.2083,0.4792,n/a,n/a,n/a,0.1774,40,see measured_metrics +CTT residual test measured,runs/ctt_residual_rollout_test_seed1,yes,measured rollout K=8,"fail:selected<47.45,support_gap>7,selector_gap>3",0.2708,0.2292,0.5417,0.7292,0.4287,0.6263,0.2292,0.3125,0.5417,n/a,n/a,n/a,0.1691,40,see measured_metrics +CTT residual test measured,runs/ctt_residual_rollout_test_seed2,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3",0.2917,0.1250,0.4583,0.7292,0.5775,0.6247,0.3125,0.3333,0.5625,n/a,n/a,n/a,0.1725,40,see measured_metrics +CTT residual validation measured,runs/ctt_residual_rollout_val16_seed0,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle",0.3125,0.2500,0.3750,0.6250,0.5139,0.2645,0.2500,0.1250,0.3750,n/a,n/a,n/a,0.1330,14,see measured_metrics +CTT residual validation measured,runs/ctt_residual_rollout_val69_seed0,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle",0.2754,0.2754,0.3913,0.6667,0.5108,0.2539,0.2899,0.1159,0.4348,n/a,n/a,n/a,0.1505,47,see measured_metrics +CTT residual validation measured,runs/ctt_residual_rollout_val69_seed1,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle",0.2899,0.1884,0.3188,0.6667,0.5993,0.3240,0.3478,0.1304,0.4348,n/a,n/a,n/a,0.1583,47,see measured_metrics +CTT residual validation measured,runs/ctt_residual_rollout_val69_seed2,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3",0.2899,0.2609,0.4203,0.6667,0.4747,0.3029,0.2464,0.1594,0.5072,n/a,n/a,n/a,0.1765,47,see measured_metrics +CTT residual train,runs/ctt_residual_smoke,yes,train,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,unknown,see run +CTT residual proxy,runs/ctt_residual_smoke_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,1.0000,1.0000,0.1010,n/a,unknown,see run +CTT residual proxy,runs/ctt_residual_val_proxy,yes,proxy,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.3478,0.7101,0.0397,n/a,47,see run +CTT residual test measured aggregate,runs/ctt_test_rollout_comparison,yes,measured rollout K=8,"fail:selected<47.45,support_gap>7,selector_gap>3",0.2847,0.2222,0.5069,0.7292,0.4833,0.5605,0.2639,0.2847,0.5278,n/a,n/a,n/a,0.1730,3,see measured_metrics +CTT residual validation measured aggregate,runs/ctt_val16_rollout_comparison,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle,seeds<3",n/a,n/a,n/a,n/a,0.5139,0.2645,n/a,n/a,0.3750,n/a,n/a,n/a,0.1330,1,see measured_metrics +CTT residual validation measured aggregate,runs/ctt_val_rollout_comparison,yes,measured rollout K=8,"fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3",0.2850,0.2415,0.3768,0.6667,0.5283,0.2936,0.2947,0.1353,0.4589,n/a,n/a,n/a,0.1618,3,see measured_metrics +task memory,runs/task_memory_val_proxy,yes,proxy baseline,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.3188,0.5362,0.0175,n/a,47,see run +local atlas,runs/local_atlas_val_proxy,yes,proxy baseline,not measured,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.4058,0.6812,0.0368,n/a,47,see run +ctt residual full,runs/ctt_val_proxy_comparison,yes,proxy aggregate gate=True,proxy=True,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.1981,0.6087,0.0296,n/a,3,aggregate over run means +ctt gated residual full,runs/ctt_val_proxy_comparison,yes,proxy aggregate gate=False,proxy=False,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.2319,0.6135,0.0527,n/a,3,aggregate over run means +Utility energy smoke,runs/utility_energy_smoke,train-only,calibration diagnostic,not deployment,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,n/a,0.1316,train split,see run diff --git a/workspace/runs/summary_ctt.md b/workspace/runs/summary_ctt.md index 180bd201945d95533a09ea8fe4f8bde5fdf747f8..12fdf75965f844afddc266be8ca73a1e887eee44 100644 --- a/workspace/runs/summary_ctt.md +++ b/workspace/runs/summary_ctt.md @@ -1,30 +1,41 @@ # CTT Summary -| Method | Status | Success | Proposal oracle | Support gap | Selector gap | OutcomePTR | PPTC@0.20 | PPTC@0.40 | NegativeNear@0.20 | Calibration ECE | Run | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| V0 residual transport diagnostic | measured diagnostic baseline | 0.3890 | 0.4435 | 0.1264 | 0.0545 | n/a | 0.2366 | 0.5269 | 0.0533 | n/a | `runs/reproduce_v0` | -| CTT gated residual | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_full_seed0` | -| CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.2319 | 0.5797 | 0.0512 | n/a | `runs/ctt_gated_residual_full_seed0_val_proxy` | -| CTT gated residual | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_full_seed1` | -| CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.2174 | 0.6232 | 0.0644 | n/a | `runs/ctt_gated_residual_full_seed1_val_proxy` | -| CTT gated residual | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_full_seed2` | -| CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.2464 | 0.6377 | 0.0424 | n/a | `runs/ctt_gated_residual_full_seed2_val_proxy` | -| CTT gated residual | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_smoke` | -| CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.5000 | 0.7500 | 0.0573 | n/a | `runs/ctt_gated_residual_smoke_proxy` | -| CTT gated residual | proxy | n/a | n/a | n/a | n/a | n/a | 0.1739 | 0.5362 | 0.0281 | n/a | `runs/ctt_gated_residual_val_proxy` | -| CTT residual train | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_full_seed0` | -| CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 0.2029 | 0.5797 | 0.0289 | n/a | `runs/ctt_residual_full_seed0_val_proxy` | -| CTT residual train | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_full_seed1` | -| CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 0.2319 | 0.5942 | 0.0410 | n/a | `runs/ctt_residual_full_seed1_val_proxy` | -| CTT residual train | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_full_seed2` | -| CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 0.1594 | 0.6522 | 0.0189 | n/a | `runs/ctt_residual_full_seed2_val_proxy` | -| CTT residual train | train | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_smoke` | -| CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 1.0000 | 1.0000 | 0.1010 | n/a | `runs/ctt_residual_smoke_proxy` | -| CTT residual proxy | proxy | n/a | n/a | n/a | n/a | n/a | 0.3478 | 0.7101 | 0.0397 | n/a | `runs/ctt_residual_val_proxy` | -| task memory | proxy baseline | n/a | n/a | n/a | n/a | n/a | 0.3188 | 0.5362 | 0.0175 | n/a | `runs/task_memory_val_proxy` | -| local atlas | proxy baseline | n/a | n/a | n/a | n/a | n/a | 0.4058 | 0.6812 | 0.0368 | n/a | `runs/local_atlas_val_proxy` | -| ctt residual full | proxy aggregate gate=True | n/a | n/a | n/a | n/a | n/a | 0.1981 | 0.6087 | 0.0296 | n/a | `runs/ctt_val_proxy_comparison` | -| ctt gated residual full | proxy aggregate gate=False | n/a | n/a | n/a | n/a | n/a | 0.2319 | 0.6135 | 0.0527 | n/a | `runs/ctt_val_proxy_comparison` | -| Utility energy smoke | calibration diagnostic | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.1316 | `runs/utility_energy_smoke` | +| Method | Status | Base | Selected | Proposal oracle | Hidden oracle | Utility support gap | Utility selector gap | Success support gap | Success selector gap | OutcomePTR | PPTC@0.20 | PPTC@0.40 | NegativeNear@0.20 | Calibration ECE | Run | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| V0 residual transport diagnostic | measured diagnostic baseline; gate: legacy diagnostic | 0.2974 | 0.3890 | 0.4435 | n/a | 0.1264 | 0.0545 | 0.1264 | 0.0545 | n/a | 0.2366 | 0.5269 | 0.0533 | n/a | `runs/reproduce_v0` | +| CTT gated residual | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_full_seed0` | +| CTT gated residual | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.2319 | 0.5797 | 0.0512 | n/a | `runs/ctt_gated_residual_full_seed0_val_proxy` | +| CTT gated residual | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_full_seed1` | +| CTT gated residual | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.2174 | 0.6232 | 0.0644 | n/a | `runs/ctt_gated_residual_full_seed1_val_proxy` | +| CTT gated residual | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_full_seed2` | +| CTT gated residual | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.2464 | 0.6377 | 0.0424 | n/a | `runs/ctt_gated_residual_full_seed2_val_proxy` | +| CTT gated residual | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_gated_residual_smoke` | +| CTT gated residual | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.5000 | 0.7500 | 0.0573 | n/a | `runs/ctt_gated_residual_smoke_proxy` | +| CTT gated residual | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.1739 | 0.5362 | 0.0281 | n/a | `runs/ctt_gated_residual_val_proxy` | +| CTT residual train | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_full_seed0` | +| CTT residual proxy | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.2029 | 0.5797 | 0.0289 | n/a | `runs/ctt_residual_full_seed0_val_proxy` | +| CTT residual train | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_full_seed1` | +| CTT residual proxy | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.2319 | 0.5942 | 0.0410 | n/a | `runs/ctt_residual_full_seed1_val_proxy` | +| CTT residual train | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_full_seed2` | +| CTT residual proxy | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.1594 | 0.6522 | 0.0189 | n/a | `runs/ctt_residual_full_seed2_val_proxy` | +| CTT residual validation measured | measured rollout K=2; gate: fail:selected<47.45,oracle<50,OutcomePTR<=V0oracle,seeds<3 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | 0.1798 | 0.0000 | 0.0000 | 0.0000 | 0.0000 | n/a | n/a | n/a | 0.4932 | `runs/ctt_residual_rollout_direct_smoke_seed0_v3` | +| CTT residual test measured | measured rollout K=8; gate: fail:selected<47.45,support_gap>7,selector_gap>3 | 0.2917 | 0.3125 | 0.5208 | 0.7292 | 0.4438 | 0.4304 | 0.2500 | 0.2083 | 0.4792 | n/a | n/a | n/a | 0.1774 | `runs/ctt_residual_rollout_test_seed0` | +| CTT residual test measured | measured rollout K=8; gate: fail:selected<47.45,support_gap>7,selector_gap>3 | 0.2708 | 0.2292 | 0.5417 | 0.7292 | 0.4287 | 0.6263 | 0.2292 | 0.3125 | 0.5417 | n/a | n/a | n/a | 0.1691 | `runs/ctt_residual_rollout_test_seed1` | +| CTT residual test measured | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3 | 0.2917 | 0.1250 | 0.4583 | 0.7292 | 0.5775 | 0.6247 | 0.3125 | 0.3333 | 0.5625 | n/a | n/a | n/a | 0.1725 | `runs/ctt_residual_rollout_test_seed2` | +| CTT residual validation measured | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle | 0.3125 | 0.2500 | 0.3750 | 0.6250 | 0.5139 | 0.2645 | 0.2500 | 0.1250 | 0.3750 | n/a | n/a | n/a | 0.1330 | `runs/ctt_residual_rollout_val16_seed0` | +| CTT residual validation measured | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle | 0.2754 | 0.2754 | 0.3913 | 0.6667 | 0.5108 | 0.2539 | 0.2899 | 0.1159 | 0.4348 | n/a | n/a | n/a | 0.1505 | `runs/ctt_residual_rollout_val69_seed0` | +| CTT residual validation measured | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle | 0.2899 | 0.1884 | 0.3188 | 0.6667 | 0.5993 | 0.3240 | 0.3478 | 0.1304 | 0.4348 | n/a | n/a | n/a | 0.1583 | `runs/ctt_residual_rollout_val69_seed1` | +| CTT residual validation measured | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3 | 0.2899 | 0.2609 | 0.4203 | 0.6667 | 0.4747 | 0.3029 | 0.2464 | 0.1594 | 0.5072 | n/a | n/a | n/a | 0.1765 | `runs/ctt_residual_rollout_val69_seed2` | +| CTT residual train | train; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | `runs/ctt_residual_smoke` | +| CTT residual proxy | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 1.0000 | 1.0000 | 0.1010 | n/a | `runs/ctt_residual_smoke_proxy` | +| CTT residual proxy | proxy; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.3478 | 0.7101 | 0.0397 | n/a | `runs/ctt_residual_val_proxy` | +| CTT residual test measured aggregate | measured rollout K=8; gate: fail:selected<47.45,support_gap>7,selector_gap>3 | 0.2847 | 0.2222 | 0.5069 | 0.7292 | 0.4833 | 0.5605 | 0.2639 | 0.2847 | 0.5278 | n/a | n/a | n/a | 0.1730 | `runs/ctt_test_rollout_comparison` | +| CTT residual validation measured aggregate | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3,OutcomePTR<=V0oracle,seeds<3 | n/a | n/a | n/a | n/a | 0.5139 | 0.2645 | n/a | n/a | 0.3750 | n/a | n/a | n/a | 0.1330 | `runs/ctt_val16_rollout_comparison` | +| CTT residual validation measured aggregate | measured rollout K=8; gate: fail:selected<47.45,oracle<50,support_gap>7,selector_gap>3 | 0.2850 | 0.2415 | 0.3768 | 0.6667 | 0.5283 | 0.2936 | 0.2947 | 0.1353 | 0.4589 | n/a | n/a | n/a | 0.1618 | `runs/ctt_val_rollout_comparison` | +| task memory | proxy baseline; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.3188 | 0.5362 | 0.0175 | n/a | `runs/task_memory_val_proxy` | +| local atlas | proxy baseline; gate: not measured | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.4058 | 0.6812 | 0.0368 | n/a | `runs/local_atlas_val_proxy` | +| ctt residual full | proxy aggregate gate=True; gate: proxy=True | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.1981 | 0.6087 | 0.0296 | n/a | `runs/ctt_val_proxy_comparison` | +| ctt gated residual full | proxy aggregate gate=False; gate: proxy=False | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.2319 | 0.6135 | 0.0527 | n/a | `runs/ctt_val_proxy_comparison` | +| Utility energy smoke | calibration diagnostic; gate: not deployment | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | n/a | 0.1316 | `runs/utility_energy_smoke` | -`n/a` means the required measured artifact does not exist yet. +`n/a` means the required measured artifact does not exist yet. Measured gates use selected >=47.45%, proposal oracle >=50%, success support gap <=7 points, success selector gap <=3 points, OutcomePTR above the V0 proposal oracle reference, and at least three train seeds. diff --git a/workspace/scripts/build_ctt_rollout_comparison.py b/workspace/scripts/build_ctt_rollout_comparison.py index cac06bcc32fba2acbcc1150cf1fc93b1b332e0c1..dee5d45942a9b9adf764aa48d3c4c5648c6f8a31 100644 --- a/workspace/scripts/build_ctt_rollout_comparison.py +++ b/workspace/scripts/build_ctt_rollout_comparison.py @@ -5,6 +5,7 @@ import argparse import json import math import re +import shutil import subprocess import sys from pathlib import Path @@ -83,10 +84,19 @@ def main(argv: list[str] | None = None) -> int: (out_dir / "metrics.json").write_text(json.dumps(_summary_payload(combined, metrics), indent=2, sort_keys=True) + "\n") (out_dir / "table.tex").write_text((out_dir / "measured_metrics" / "table.tex").read_text()) (out_dir / "report.md").write_text(_report(combined, metrics) + "\n") + for filename in ("metrics_by_task.json", "metrics_by_seed.json"): + shutil.copyfile(out_dir / "measured_metrics" / filename, out_dir / filename) (out_dir / "command.txt").write_text( "python scripts/build_ctt_rollout_comparison.py " + " ".join(sys.argv[1:]) + "\n" ) (out_dir / "git_hash.txt").write_text(_run(["git", "rev-parse", "HEAD"]) + "\n") + (out_dir / "data_hash.txt").write_text(str(combined.get("target_content_hash") or "") + "\n") + (out_dir / "split_hash.txt").write_text(str(combined.get("target_split_hash") or "") + "\n") + (out_dir / "source_data_hash.txt").write_text(str(combined.get("source_content_hash") or "") + "\n") + (out_dir / "train.log").write_text("comparison artifact; source runs trained/evaluated separately\n") + (out_dir / "eval.log").write_text( + "\n".join(f"{run_dir}: measured_candidates.json" for run_dir in combined["run_dirs"]) + "\n" + ) print(json.dumps({"out_dir": str(out_dir), "runs": len(run_dirs), "rows": len(combined_rows)}, indent=2)) return 0 @@ -99,6 +109,7 @@ def _summary_payload(combined: dict[str, Any], metrics: dict[str, Any]) -> dict[ "train_seeds": combined["train_seeds"], "num_rows": combined["num_rows"], "summary": metrics.get("summary", {}), + "success_summary": _success_summary(combined.get("rows", []), k=int(combined["k"])), "source_content_hash": combined.get("source_content_hash"), "target_content_hash": combined.get("target_content_hash"), "target_split_hash": combined.get("target_split_hash"), @@ -107,8 +118,9 @@ def _summary_payload(combined: dict[str, Any], metrics: dict[str, Any]) -> dict[ def _report(combined: dict[str, Any], metrics: dict[str, Any]) -> str: summary = metrics.get("summary", {}) + split = _split_name(combined) lines = [ - "# CTT Validation Measured Rollout Comparison", + f"# CTT {split.title()} Measured Rollout Comparison", "", f"Runs: `{len(combined['run_dirs'])}`", f"Rows: `{combined['num_rows']}`", @@ -124,6 +136,26 @@ def _report(combined: dict[str, Any], metrics: dict[str, Any]) -> str: f"[{_fmt(micro.get('low'))}, {_fmt(micro.get('high'))}] |" ) lines.append("") + success = _success_summary(combined.get("rows", []), k=int(combined["k"])) + lines.extend( + [ + "| Success/Utility | Mean |", + "| --- | ---: |", + f"| base_success_rate | {_fmt(success.get('base_success_rate'))} |", + f"| selected_success_rate | {_fmt(success.get('selected_success_rate'))} |", + f"| proposal_oracle_success_rate | {_fmt(success.get('proposal_oracle_success_rate'))} |", + f"| hidden_chart_oracle_success_rate | {_fmt(success.get('hidden_chart_oracle_success_rate'))} |", + f"| selected_success_gain_over_base | {_fmt(success.get('selected_success_gain_over_base'))} |", + f"| proposal_oracle_success_gain_over_base | {_fmt(success.get('proposal_oracle_success_gain_over_base'))} |", + f"| success_support_gap | {_fmt(success.get('success_support_gap'))} |", + f"| success_selector_gap | {_fmt(success.get('success_selector_gap'))} |", + f"| base_utility_mean | {_fmt(success.get('base_utility_mean'))} |", + f"| selected_utility_mean | {_fmt(success.get('selected_utility_mean'))} |", + f"| proposal_oracle_utility_mean | {_fmt(success.get('proposal_oracle_utility_mean'))} |", + f"| hidden_chart_oracle_utility_mean | {_fmt(success.get('hidden_chart_oracle_utility_mean'))} |", + "", + ] + ) lines.append("These are measured generated-candidate rollouts, not PPTC proxies.") lines.append("") lines.append("Run dirs:") @@ -137,6 +169,15 @@ def _seed_from_path(path: Path) -> str: return match.group(1) if match else path.name +def _split_name(combined: dict[str, Any]) -> str: + text = " ".join(str(item) for item in combined.get("run_dirs", [])) + if "test" in text: + return "test" + if "val" in text or "validation" in text: + return "validation" + return "measured" + + def _first(payloads: list[dict[str, Any]], key: str) -> Any: for payload in payloads: value = payload.get(key) @@ -145,6 +186,88 @@ def _first(payloads: list[dict[str, Any]], key: str) -> Any: return None +def _success_summary(rows: list[dict[str, Any]], *, k: int) -> dict[str, Any]: + base_success = [] + selected_success = [] + oracle_success = [] + base_utility = [] + selected_utility = [] + oracle_utility = [] + hidden_oracle_utility = [] + hidden_oracle_success = [] + success_support_gap = [] + success_selector_gap = [] + selected_success_gain = [] + proposal_oracle_success_gain = [] + restore_errors = [] + for row in rows: + generated_utilities = [float(value) for value in row.get("generated_utilities", [])[:k]] + generated_success = [bool(value) for value in row.get("candidate_success", [])[:k]] + selected_index = int(row.get("selected_index", 0)) + selected_success_value: float | None = None + proposal_oracle_success_value: float | None = None + base_success_value: float | None = None + if "base_success" in row: + base_success_value = float(bool(row["base_success"])) + base_success.append(base_success_value) + if selected_index < len(generated_success): + selected_success_value = float(generated_success[selected_index]) + selected_success.append(selected_success_value) + if generated_success: + proposal_oracle_success_value = float(any(generated_success)) + oracle_success.append(proposal_oracle_success_value) + if "base_utility" in row: + base_utility.append(float(row["base_utility"])) + if selected_index < len(generated_utilities): + selected_utility.append(generated_utilities[selected_index]) + if generated_utilities: + oracle_utility.append(max(generated_utilities)) + hidden = [float(value) for value in row.get("hidden_chart_utilities", [])] + if hidden: + hidden_oracle_utility.append(max(hidden)) + hidden_success_value = float(any(value >= 1.0 for value in hidden)) + hidden_oracle_success.append(hidden_success_value) + if proposal_oracle_success_value is not None: + success_support_gap.append( + max(0.0, hidden_success_value - proposal_oracle_success_value) + ) + if ( + proposal_oracle_success_value is not None + and selected_success_value is not None + ): + success_selector_gap.append( + max(0.0, proposal_oracle_success_value - selected_success_value) + ) + if base_success_value is not None and selected_success_value is not None: + selected_success_gain.append(selected_success_value - base_success_value) + if base_success_value is not None and proposal_oracle_success_value is not None: + proposal_oracle_success_gain.append( + proposal_oracle_success_value - base_success_value + ) + if "restore_error" in row: + restore_errors.append(float(row["restore_error"])) + return { + "base_success_rate": _mean(base_success), + "selected_success_rate": _mean(selected_success), + "proposal_oracle_success_rate": _mean(oracle_success), + "hidden_chart_oracle_success_rate": _mean(hidden_oracle_success), + "selected_success_gain_over_base": _mean(selected_success_gain), + "proposal_oracle_success_gain_over_base": _mean(proposal_oracle_success_gain), + "success_support_gap": _mean(success_support_gap), + "success_selector_gap": _mean(success_selector_gap), + "base_utility_mean": _mean(base_utility), + "selected_utility_mean": _mean(selected_utility), + "proposal_oracle_utility_mean": _mean(oracle_utility), + "hidden_chart_oracle_utility_mean": _mean(hidden_oracle_utility), + "max_restore_error": max(restore_errors) if restore_errors else None, + } + + +def _mean(values: list[float]) -> float | None: + clean = [float(value) for value in values if math.isfinite(float(value))] + return sum(clean) / len(clean) if clean else None + + def _fmt(value: Any) -> str: if not isinstance(value, (int, float)) or not math.isfinite(float(value)): return "n/a" diff --git a/workspace/scripts/eval_ctt_generated_rollout.py b/workspace/scripts/eval_ctt_generated_rollout.py index 1008d654d98ffcdfe4de1408b9b82c38361cb1d5..1a56d6a829913db229ecd6b7cafc4c2b7bc0d71a 100644 --- a/workspace/scripts/eval_ctt_generated_rollout.py +++ b/workspace/scripts/eval_ctt_generated_rollout.py @@ -5,6 +5,7 @@ import argparse import json import math import pickle +import shutil import subprocess import sys from dataclasses import dataclass @@ -218,13 +219,14 @@ def main(argv: list[str] | None = None) -> int: measured_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") (out_dir / "report.md").write_text(_report(payload) + "\n") + metrics_dir = out_dir / "measured_metrics" if not args.skip_metrics: eval_metrics_main( [ "--input", str(measured_path), "--out-dir", - str(out_dir / "measured_metrics"), + str(metrics_dir), "--mode", "measured", "--k", @@ -233,6 +235,13 @@ def main(argv: list[str] | None = None) -> int: str(args.bootstrap_samples), ] ) + _write_required_artifacts( + out_dir, + payload, + source_index=source_index, + target_index=target_index, + metrics_dir=metrics_dir if metrics_dir.exists() else None, + ) print( json.dumps( @@ -552,48 +561,15 @@ def rollout_generated_cases( progress_value + (1.0 if success_value else 0.0) for progress_value, success_value in zip(progress, success, strict=True) ] - base_utility = float(utilities[0]) - generated_utilities = [float(value) for value in utilities[1:]] - predicted_scores = [float(proposal.score) for proposal in proposals] rows.append( - { - "chart_id": target.chart_id, - "group_id": target.chart_id, - "task_id": target.task_id, - "seed": target.seed, - "state_hash": target.state_hash, - "instruction": target.instruction, - "candidates_evaluated": True, - "selected_index": 0, - "base_utility": base_utility, - "stored_base_utility": target.stored_base_utility, - "generated_utilities": generated_utilities, - "hidden_chart_utilities": target.hidden_utilities, - "hidden_candidate_types": target.hidden_candidate_types, - "predicted_scores": predicted_scores, - "generated_tangents": [ - proposal.tangent.astype(float).tolist() for proposal in proposals - ], - "positive_tangents": target.positive_tangents.astype(float).tolist(), - "negative_tangents": target.negative_tangents.astype(float).tolist(), - "candidate_types": [ - f"ctt_transport_rank{proposal.source_rank}" - for proposal in proposals - ], - "candidate_source_chart_ids": [ - proposal.source_chart_id for proposal in proposals - ], - "candidate_source_task_ids": [ - proposal.source_task_id for proposal in proposals - ], - "candidate_progress": progress[1:], - "candidate_success": success[1:], - "base_progress": progress[0], - "base_success": success[0], - "restore_error": float(restore_error), - "num_generated": len(proposals), - "num_executed_including_base": valid, - } + _measured_row_from_rollout( + target, + proposals, + progress=progress, + success=success, + utilities=utilities, + restore_error=float(restore_error), + ) ) finally: env.close() @@ -701,6 +677,34 @@ def _measured_row_from_rollout( "generated_utilities": generated_utilities, "hidden_chart_utilities": target.hidden_utilities, "hidden_candidate_types": target.hidden_candidate_types, + "outcome_vector_schema": [ + "success", + "progress", + "contact_quality", + "safety_violation", + "stage_progress", + "smoothness", + "recovery", + "utility_scalar", + ], + "base_outcome": _outcome_payload( + success=success[0], + progress=progress[0], + utility=utilities[0], + ), + "candidate_outcomes": [ + _outcome_payload( + success=success_value, + progress=progress_value, + utility=utility, + ) + for success_value, progress_value, utility in zip( + success[1:], + progress[1:], + utilities[1:], + strict=True, + ) + ], "predicted_scores": predicted_scores, "generated_tangents": [ proposal.tangent.astype(float).tolist() for proposal in proposals @@ -722,6 +726,19 @@ def _measured_row_from_rollout( } +def _outcome_payload(*, success: bool, progress: float, utility: float) -> dict[str, Any]: + return { + "success": bool(success), + "progress": float(progress), + "contact_quality": None, + "safety_violation": None, + "stage_progress": None, + "smoothness": None, + "recovery": None, + "utility_scalar": float(utility), + } + + def _uses_single_env_cpu_backend(sim_backend: Any) -> bool: value = str(sim_backend or "").lower() return value in {"cpu", "physx_cpu"} or value.endswith("_cpu") @@ -860,6 +877,131 @@ def _write_run_provenance(out_dir: Path, args: argparse.Namespace) -> None: pass +def _write_required_artifacts( + out_dir: Path, + payload: dict[str, Any], + *, + source_index: dict[str, Any], + target_index: dict[str, Any], + metrics_dir: Path | None, +) -> None: + (out_dir / "data_hash.txt").write_text(str(target_index.get("content_hash", "")) + "\n") + (out_dir / "split_hash.txt").write_text(str(target_index.get("split_hash", "")) + "\n") + (out_dir / "source_data_hash.txt").write_text(str(source_index.get("content_hash", "")) + "\n") + (out_dir / "source_split_hash.txt").write_text(str(source_index.get("split_hash", "")) + "\n") + (out_dir / "train.log").write_text( + "CTT checkpoint trained separately; rollout used checkpoint=" + f"{payload.get('checkpoint', '')}\n" + ) + run_log = out_dir / "run.log" + (out_dir / "eval.log").write_text(run_log.read_text() if run_log.exists() else "eval log unavailable\n") + summary = { + "report_type": payload.get("report_type"), + "schema_version": payload.get("schema_version"), + "k": payload.get("k"), + "checkpoint": payload.get("checkpoint"), + "source_content_hash": payload.get("source_content_hash"), + "source_split_hash": payload.get("source_split_hash"), + "target_content_hash": payload.get("target_content_hash"), + "target_split_hash": payload.get("target_split_hash"), + "num_rows": len(payload.get("rows", [])), + "success_summary": _success_summary(payload.get("rows", []), k=int(payload.get("k", 0))), + } + if metrics_dir is not None: + metric_path = metrics_dir / "metrics.json" + if metric_path.exists(): + metrics = json.loads(metric_path.read_text()) + summary["measured_metric_summary"] = metrics.get("summary", {}) + for filename in ("metrics_by_task.json", "metrics_by_seed.json", "table.tex"): + src = metrics_dir / filename + if src.exists(): + shutil.copyfile(src, out_dir / filename) + (out_dir / "metrics.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + for filename in ("metrics_by_task.json", "metrics_by_seed.json", "table.tex"): + path = out_dir / filename + if not path.exists(): + path.write_text("{}\n" if filename.endswith(".json") else "% metrics not computed\n") + + +def _success_summary(rows: list[dict[str, Any]], *, k: int) -> dict[str, Any]: + if k <= 0: + k = 10**9 + base_success = [] + selected_success = [] + oracle_success = [] + base_utility = [] + selected_utility = [] + oracle_utility = [] + hidden_oracle_utility = [] + hidden_oracle_success = [] + success_support_gap = [] + success_selector_gap = [] + selected_success_gain = [] + proposal_oracle_success_gain = [] + restore_errors = [] + for row in rows: + generated_utilities = [float(value) for value in row.get("generated_utilities", [])[:k]] + generated_success = [bool(value) for value in row.get("candidate_success", [])[:k]] + selected_index = int(row.get("selected_index", 0)) + selected_success_value: float | None = None + proposal_oracle_success_value: float | None = None + base_success_value: float | None = None + if "base_success" in row: + base_success_value = float(bool(row["base_success"])) + base_success.append(base_success_value) + if selected_index < len(generated_success): + selected_success_value = float(generated_success[selected_index]) + selected_success.append(selected_success_value) + if generated_success: + proposal_oracle_success_value = float(any(generated_success)) + oracle_success.append(proposal_oracle_success_value) + if "base_utility" in row: + base_utility.append(float(row["base_utility"])) + if selected_index < len(generated_utilities): + selected_utility.append(float(generated_utilities[selected_index])) + if generated_utilities: + oracle_utility.append(max(generated_utilities)) + hidden = [float(value) for value in row.get("hidden_chart_utilities", [])] + if hidden: + hidden_oracle_utility.append(max(hidden)) + hidden_success_value = float(any(value >= 1.0 for value in hidden)) + hidden_oracle_success.append(hidden_success_value) + if proposal_oracle_success_value is not None: + success_support_gap.append( + max(0.0, hidden_success_value - proposal_oracle_success_value) + ) + if ( + proposal_oracle_success_value is not None + and selected_success_value is not None + ): + success_selector_gap.append( + max(0.0, proposal_oracle_success_value - selected_success_value) + ) + if base_success_value is not None and selected_success_value is not None: + selected_success_gain.append(selected_success_value - base_success_value) + if base_success_value is not None and proposal_oracle_success_value is not None: + proposal_oracle_success_gain.append( + proposal_oracle_success_value - base_success_value + ) + if "restore_error" in row: + restore_errors.append(float(row["restore_error"])) + return { + "base_success_rate": _mean(base_success), + "selected_success_rate": _mean(selected_success), + "proposal_oracle_success_rate": _mean(oracle_success), + "hidden_chart_oracle_success_rate": _mean(hidden_oracle_success), + "selected_success_gain_over_base": _mean(selected_success_gain), + "proposal_oracle_success_gain_over_base": _mean(proposal_oracle_success_gain), + "success_support_gap": _mean(success_support_gap), + "success_selector_gap": _mean(success_selector_gap), + "base_utility_mean": _mean(base_utility), + "selected_utility_mean": _mean(selected_utility), + "proposal_oracle_utility_mean": _mean(oracle_utility), + "hidden_chart_oracle_utility_mean": _mean(hidden_oracle_utility), + "max_restore_error": max(restore_errors) if restore_errors else None, + } + + def _append_log(path: Path | None, message: str) -> None: if path is None: return diff --git a/workspace/scripts/eval_metrics.py b/workspace/scripts/eval_metrics.py index 569f07d798d4e224b22f9c460832ef19384a3b47..2bac845bfbc9ea5ee7f0e40491199406ecedd1f7 100644 --- a/workspace/scripts/eval_metrics.py +++ b/workspace/scripts/eval_metrics.py @@ -129,12 +129,16 @@ def _measured_row(row: dict[str, Any], *, k: int, epsilon: float) -> dict[str, A raise MetricInputError("measured rows require generated_utilities") selected_index = int(row.get("selected_index", 0)) hidden = _numbers(row, "hidden_chart_utilities", required=False) + candidate_success = _bool_numbers(row, "candidate_success", required=False) + base_success = _optional_bool(row.get("base_success")) selected_utility = utilities[selected_index] prefix = utilities[:k] output = _base_row(row, mode="measured") + base_utility = _number(row, "base_utility") + proposal_oracle_utility = max(prefix) output[f"outcome_ptr_at_{k}"] = outcome_ptr_at_k( utilities, - _number(row, "base_utility"), + base_utility, epsilon=epsilon, k=k, candidates_evaluated=True, @@ -146,12 +150,56 @@ def _measured_row(row: dict[str, Any], *, k: int, epsilon: float) -> dict[str, A candidates_evaluated=True, ) output[f"branch_car_at_{k}"] = branch_car(max(prefix), selected_utility) + output["base_utility"] = base_utility + output[f"selected_utility_at_{k}"] = selected_utility + output[f"proposal_oracle_utility_at_{k}"] = proposal_oracle_utility + output[f"selected_utility_gain_over_base_at_{k}"] = selected_utility - base_utility + output[f"proposal_oracle_utility_gain_over_base_at_{k}"] = ( + proposal_oracle_utility - base_utility + ) + if base_success is not None: + output["base_success"] = float(base_success) + if candidate_success: + success_prefix = candidate_success[:k] + selected_success = float(success_prefix[selected_index]) + proposal_oracle_success = float(any(success_prefix)) + output[f"selected_success_at_{k}"] = selected_success + output[f"proposal_oracle_success_at_{k}"] = proposal_oracle_success + if base_success is not None: + output[f"selected_success_gain_over_base_at_{k}"] = ( + selected_success - float(base_success) + ) + output[f"proposal_oracle_success_gain_over_base_at_{k}"] = ( + proposal_oracle_success - float(base_success) + ) if hidden: + hidden_oracle_utility = max(hidden) output[f"support_gap_at_{k}"] = measured_support_gap( - max(hidden), + hidden_oracle_utility, max(prefix), candidates_evaluated=True, ) + output[f"hidden_chart_oracle_utility_at_{k}"] = hidden_oracle_utility + output[f"total_car_to_hidden_at_{k}"] = branch_car( + hidden_oracle_utility, + selected_utility, + ) + if candidate_success: + hidden_oracle_success = float(any(value >= 1.0 for value in hidden)) + output[f"hidden_chart_oracle_success_at_{k}"] = hidden_oracle_success + output[f"success_support_gap_at_{k}"] = max( + 0.0, + hidden_oracle_success - output[f"proposal_oracle_success_at_{k}"], + ) + output[f"success_selector_gap_at_{k}"] = max( + 0.0, + output[f"proposal_oracle_success_at_{k}"] + - output[f"selected_success_at_{k}"], + ) + output[f"success_total_car_to_hidden_at_{k}"] = max( + 0.0, + hidden_oracle_success - output[f"selected_success_at_{k}"], + ) predicted = _numbers(row, "predicted_scores", required=False) if predicted and len(predicted) >= len(utilities): ece = pairwise_causal_dominance_ece(predicted[: len(utilities)], utilities) @@ -221,6 +269,23 @@ def _number(row: dict[str, Any], key: str) -> float: return float(row[key]) +def _bool_numbers(row: dict[str, Any], key: str, *, required: bool = True) -> list[bool]: + values = row.get(key) + if values is None: + if required: + raise MetricInputError(f"row requires {key}") + return [] + if not isinstance(values, list): + raise MetricInputError(f"{key} must be a list") + return [bool(value) for value in values] + + +def _optional_bool(value: Any) -> bool | None: + if value is None: + return None + return bool(value) + + def _matrix(row: dict[str, Any], key: str, *, required: bool = True) -> list[list[float]]: values = row.get(key) if values is None: